diff --git a/.gitattributes b/.gitattributes index 34e2bb35794d14dd5f851c967b42f95255b7464d..28577f0416eb04985466336e4db15180d966f08c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -108,3 +108,45 @@ tmmu86ea/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text tmmu86ea/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text tmmu86ea/episode_rosbags/episode_2_2025-01-03-20-11-57.bag filter=lfs diff=lfs merge=lfs -text 3cjc38bu/episode_rosbags/episode_2_2025-01-04-20-55-09.bag filter=lfs diff=lfs merge=lfs -text +3cjc38bu/episode_rosbags/episode_1_2025-01-04-20-54-27.bag filter=lfs diff=lfs merge=lfs -text +3cjc38bu/episode_rosbags/episode_3_2025-01-04-20-56-00.bag filter=lfs diff=lfs merge=lfs -text +3cjc38bu/episode_rosbags/episode_4_2025-01-04-20-56-47.bag filter=lfs diff=lfs merge=lfs -text +3cjc38bu/episode_rosbags/episode_0_2025-01-04-20-53-42.bag filter=lfs diff=lfs merge=lfs -text +u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/episode_rosbags/episode_2_2025-01-03-22-42-16.bag filter=lfs diff=lfs merge=lfs -text +u0i99mq0/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/episode_rosbags/episode_0_2025-01-03-22-45-42.bag filter=lfs diff=lfs merge=lfs -text +amw07z8o/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/episode_rosbags/episode_4_2025-01-04-22-39-10.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/episode_rosbags/episode_1_2025-01-03-20-06-06.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/episode_rosbags/episode_0_2025-01-03-20-05-17.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/episode_rosbags/episode_2_2025-01-03-20-06-47.bag filter=lfs diff=lfs merge=lfs -text +yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/episode_rosbags/episode_3_2025-01-03-20-07-37.bag filter=lfs diff=lfs merge=lfs -text +yxg6asez/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/episode_rosbags/episode_1_2025-01-04-20-49-45.bag filter=lfs diff=lfs merge=lfs -text diff --git a/3cjc38bu/episode_rosbags/episode_0_2025-01-04-20-53-42.bag b/3cjc38bu/episode_rosbags/episode_0_2025-01-04-20-53-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..ac8577be794dbcce347605a8e013264a0dd83912 --- /dev/null +++ b/3cjc38bu/episode_rosbags/episode_0_2025-01-04-20-53-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aec26ec773700a783e82caae052916374e4b6a3a6cbce5ee49cadc1e2744bb9 +size 1417380431 diff --git a/3cjc38bu/episode_rosbags/episode_1_2025-01-04-20-54-27.bag b/3cjc38bu/episode_rosbags/episode_1_2025-01-04-20-54-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..954f50424bafdde2a0d0eee35ca4e68515755de7 --- /dev/null +++ b/3cjc38bu/episode_rosbags/episode_1_2025-01-04-20-54-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b48b4b7f95f187c5625ce40b797b0f78619b768d792d7393b3f7668e1903d9 +size 1403442241 diff --git a/3cjc38bu/episode_rosbags/episode_3_2025-01-04-20-56-00.bag b/3cjc38bu/episode_rosbags/episode_3_2025-01-04-20-56-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..60d6e334d548e031e38ad717036b016b35465767 --- /dev/null +++ b/3cjc38bu/episode_rosbags/episode_3_2025-01-04-20-56-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:768f3172252c532ce5ccf9358c94b66430a329522933035f65b8108a4c30f5e2 +size 1409953111 diff --git a/3cjc38bu/episode_rosbags/episode_4_2025-01-04-20-56-47.bag b/3cjc38bu/episode_rosbags/episode_4_2025-01-04-20-56-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..fa15d97df494dfedbbaa3be546df18f7f6a40b81 --- /dev/null +++ b/3cjc38bu/episode_rosbags/episode_4_2025-01-04-20-56-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e181033a06365fce4c382d0ebfdbfe7f20290a4534d63dc313c2bb0378f827fc +size 1412650876 diff --git a/52k33ku2/episode_rosbags/episode_0_2025-01-03-22-45-42.bag b/52k33ku2/episode_rosbags/episode_0_2025-01-03-22-45-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..cf6fe36d65667a10a79fcaaaeb7d332ed388f973 --- /dev/null +++ b/52k33ku2/episode_rosbags/episode_0_2025-01-03-22-45-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84d70608b70092c55134427496671327ec71b497a4359fb44dccf09f20b3715 +size 1409778739 diff --git a/amw07z8o/.hydra/config.yaml b/amw07z8o/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/amw07z8o/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/amw07z8o/.hydra/hydra.yaml b/amw07z8o/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a4c0391a06c5a7da0f165d10356382174daa9da --- /dev/null +++ b/amw07z8o/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/amw07z8o/.hydra/overrides.yaml b/amw07z8o/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/amw07z8o/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/amw07z8o/episode_rosbags/aligned_depth_to_color_K.npy b/amw07z8o/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/amw07z8o/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/amw07z8o/episode_rosbags/cam_tf_world.npy b/amw07z8o/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/amw07z8o/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/amw07z8o/episode_rosbags/color_K.npy b/amw07z8o/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/amw07z8o/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/amw07z8o/episode_rosbags/depth_K.npy b/amw07z8o/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/amw07z8o/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/amw07z8o/episode_rosbags/episode_4_2025-01-04-22-39-10.bag b/amw07z8o/episode_rosbags/episode_4_2025-01-04-22-39-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..61a9db6af3f8424b4508e346ea292a36cff934ac --- /dev/null +++ b/amw07z8o/episode_rosbags/episode_4_2025-01-04-22-39-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08c7e77a3d9d32f94ed24f7a3446fdb0ae7c2783daf504b9e74e60f5d252a630 +size 1411059121 diff --git a/amw07z8o/eval_robot.log b/amw07z8o/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..e3c08b704cf2b6b58d949e1742d2484a2829f146 --- /dev/null +++ b/amw07z8o/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:35:09,038][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:35:09,041][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/amw07z8o/eval_video/0_eval.mp4 b/amw07z8o/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4785ad9530c173cf6a520c308069fb377c94e9c0 --- /dev/null +++ b/amw07z8o/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17dfc8269beff249beeaa68babd76191e5205d080336e6c7317121a355b58d2e +size 1201547 diff --git a/amw07z8o/eval_video/1_eval.mp4 b/amw07z8o/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92a134ce6a2c2a0894be481c99da0bd6b2803058 --- /dev/null +++ b/amw07z8o/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b4da146e47f9bb9dd381e6c5e125d416e0fca2c5a980fa101eaa6445138b807 +size 1197063 diff --git a/amw07z8o/eval_video/2_eval.mp4 b/amw07z8o/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ac6df58df8aeafba688205104d851efd9e48509 --- /dev/null +++ b/amw07z8o/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a498ba0c97696b225f3555a898d21e754da3884a554f9a87bddfe6e7c8125c15 +size 1211751 diff --git a/amw07z8o/eval_video/3_eval.mp4 b/amw07z8o/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78638d5060651ade632be3d40cdbcde8f74dbcb3 --- /dev/null +++ b/amw07z8o/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b597141eccbb7769cb09cb50fb9673419331351d8fa33477f4377d2ed804a388 +size 1195007 diff --git a/amw07z8o/eval_video/4_eval.mp4 b/amw07z8o/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..287832894886a538330c9b9f3afdefff1a259734 --- /dev/null +++ b/amw07z8o/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1da214e6639c86f006cb2279112de21986fbe0aed182713e0621d957827c9544 +size 1241031 diff --git a/amw07z8o/tb/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 b/amw07z8o/tb/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 new file mode 100644 index 0000000000000000000000000000000000000000..df933161228b638bb785405a019f777d7ad22e1a --- /dev/null +++ b/amw07z8o/tb/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eefbac36f820ca6887036431476c2a4ff2ddd30e04c95446982a33382f1f3af5 +size 6539 diff --git a/amw07z8o/wandb/debug-internal.log b/amw07z8o/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c30f10401ccb487f1a3a2dd6731e1b927b4c192a --- /dev/null +++ b/amw07z8o/wandb/debug-internal.log @@ -0,0 +1,790 @@ +2025-01-04 22:35:14,865 INFO StreamThr :2039437 [internal.py:wandb_internal():85] W&B internal server running at pid: 2039437, started at: 2025-01-04 22:35:14.864650 +2025-01-04 22:35:14,865 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:35:14,883 INFO WriterThread:2039437 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb +2025-01-04 22:35:14,883 DEBUG SenderThread:2039437 [sender.py:send():379] send: header +2025-01-04 22:35:14,889 DEBUG SenderThread:2039437 [sender.py:send():379] send: run +2025-01-04 22:35:15,137 INFO SenderThread:2039437 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files +2025-01-04 22:35:15,137 INFO SenderThread:2039437 [sender.py:_start_run_threads():1188] run started: amw07z8o with start time 1736048114.865285 +2025-01-04 22:35:15,152 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:35:15,153 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:35:15,225 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:35:15,230 DEBUG HandlerThread:2039437 [system_info.py:__init__():26] System info init +2025-01-04 22:35:15,231 DEBUG HandlerThread:2039437 [system_info.py:__init__():41] System info init done +2025-01-04 22:35:15,231 INFO HandlerThread:2039437 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:35:15,231 INFO SystemMonitor:2039437 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:35:15,231 INFO HandlerThread:2039437 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:35:15,232 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:35:15,232 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:35:15,233 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:35:15,233 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:35:15,234 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:35:15,244 DEBUG HandlerThread:2039437 [system_info.py:probe():152] Probing system +2025-01-04 22:35:15,246 DEBUG HandlerThread:2039437 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_info.py:probe():200] Probing system done +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:35:15.245061', 'startedAt': '2025-01-05T03:35:14.853349', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.794625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.037, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 774.6898498535156}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:35:15,251 INFO HandlerThread:2039437 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:35:15,251 INFO HandlerThread:2039437 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_info.py:_save_code():44] Saving code +2025-01-04 22:35:15,252 DEBUG HandlerThread:2039437 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:35:15,252 DEBUG HandlerThread:2039437 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:35:15,315 DEBUG HandlerThread:2039437 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:35:15,315 INFO HandlerThread:2039437 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:35:15,318 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:35:15,318 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:35:15,318 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:35:15,319 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:35:15,564 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/hca1n5xm-wandb-metadata.json +2025-01-04 22:35:15,566 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:35:15,566 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:35:15,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:35:15,567 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:35:15,568 INFO wandb-upload_2:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/p3bg4nbc-diff.patch +2025-01-04 22:35:15,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:15,571 INFO HandlerThread:2039437 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/tb" + save: true +} +control { + relay_id: "e3c9300cb5724fd08f0cdf614d01cd70" +} +_info { + stream_id: "amw07z8o" +} + +2025-01-04 22:35:15,572 DEBUG HandlerThread:2039437 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:35:15,597 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/8pqe5g39-code/FISH/eval_robot.py +2025-01-04 22:35:15,683 DEBUG SenderThread:2039437 [sender.py:send():379] send: telemetry +2025-01-04 22:35:15,683 DEBUG SenderThread:2039437 [sender.py:send():379] send: tbrecord +2025-01-04 22:35:15,684 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:35:15,684 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 with policy live +2025-01-04 22:35:15,859 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/8dz9soy1-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:35:16,139 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:35:16,139 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt +2025-01-04 22:35:16,139 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code +2025-01-04 22:35:16,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:17,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:18,138 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:18,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:19,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:20,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:20,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:21,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:22,429 DEBUG SenderThread:2039437 [sender.py:send():379] send: config +2025-01-04 22:35:22,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:23,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:24,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:24,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:25,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:26,141 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:26,151 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:26,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:27,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:28,141 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:28,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:29,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:30,142 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:30,566 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:35:30,567 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:35:30,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:31,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:31,666 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:32,142 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:32,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:33,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:34,143 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:34,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:35,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:36,144 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:36,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:36,722 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:37,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:38,144 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:38,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:39,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:40,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:41,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:42,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:42,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:43,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:44,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:45,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:35:45,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:35:45,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:46,572 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:47,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:47,720 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:48,147 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml +2025-01-04 22:35:48,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:49,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:50,148 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:50,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:51,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:52,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:53,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:53,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:54,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:55,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:56,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:57,575 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:58,575 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:59,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:59,575 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:00,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:00,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:00,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:01,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:02,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:03,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:04,579 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:04,671 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:04,713 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:36:04,714 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:36:04,714 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 with policy now +2025-01-04 22:36:04,714 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:36:04,714 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:36:04,714 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:36:04,961 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/sdieoeko-media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos +2025-01-04 22:36:05,411 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/k1hw2xsb-media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:36:05,581 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:06,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:06,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:07,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:08,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:09,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:10,153 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:10,237 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:10,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:11,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:12,154 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:12,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:13,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:14,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:15,234 DEBUG SystemMonitor:2039437 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:36:15,235 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:36:15,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:15,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:15,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:15,663 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:16,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:17,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:18,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:19,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:20,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:21,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:21,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:22,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:23,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:24,158 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:36:24,390 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/kpdx4oln-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:36:24,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:25,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:26,159 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:26,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:26,585 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:27,585 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:28,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:29,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:30,160 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:30,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:30,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:30,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:31,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:31,624 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:32,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:33,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:34,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:35,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:36,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:37,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:37,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:38,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:39,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:40,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:41,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:42,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:43,572 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:43,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:44,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:45,236 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:36:45,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:45,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:45,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:46,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:47,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:48,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:48,651 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:49,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:50,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:51,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:52,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:53,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:54,565 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:54,605 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:55,591 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:55,721 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:36:55,730 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:36:55,730 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 with policy now +2025-01-04 22:36:55,730 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:36:55,731 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:36:55,731 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:36:55,987 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/r6xevbwr-media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:36:56,168 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:36:56,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:56,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:36:56,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:36:56,423 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/gfouxrg2-media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:36:56,595 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:57,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:58,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:58,592 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:59,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:00,170 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:00,261 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:00,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:00,570 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:00,617 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:01,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:02,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:03,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:03,980 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:37:03,980 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:37:03,985 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:37:04,171 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:37:04,171 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:37:04,171 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:04,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:05,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:05,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:06,172 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:06,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:07,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:08,173 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:08,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:09,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:10,173 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:10,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:11,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:11,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:12,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:13,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:14,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:15,236 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:37:15,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:15,569 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:15,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:16,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:16,721 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:17,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:18,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:19,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:20,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:21,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:22,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:22,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:23,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:24,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:25,595 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:26,595 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:27,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:27,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:28,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:29,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:30,181 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:30,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:30,569 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:30,617 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:31,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:32,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:32,667 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:33,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:34,597 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:35,598 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:35,751 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:37:35,751 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 with policy now +2025-01-04 22:37:35,754 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:37:35,755 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:37:35,755 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:37:35,756 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:37:36,000 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/csni4vgu-media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:37:36,431 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/yogqmswq-media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:37:36,616 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:37,600 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:37,777 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:38,184 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:38,605 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:39,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:40,184 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:40,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:41,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:42,600 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:43,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:43,769 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:44,186 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:44,602 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:45,237 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:37:45,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:45,570 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:45,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:46,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:47,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:48,608 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:49,611 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:49,683 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:50,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:51,610 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:52,611 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:53,611 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:54,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:55,579 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:55,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:56,612 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:57,612 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:58,617 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:59,614 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:00,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:00,570 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:00,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:00,655 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:01,614 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:02,615 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:03,616 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:04,619 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:05,620 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:06,579 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:06,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:07,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:08,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:09,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:10,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:11,585 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:11,621 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:12,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:13,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:14,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:15,196 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:38:15,237 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:38:15,424 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/ydkdtee9-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:38:15,572 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:15,573 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:15,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:15,850 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:38:15,850 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:38:15,850 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:38:16,196 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:38:16,196 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:16,624 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:17,037 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:17,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:18,197 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:18,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:19,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:20,198 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:20,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:21,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:22,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:22,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:23,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:24,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:25,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:26,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:27,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:28,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:28,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:29,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:30,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:30,571 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:30,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:31,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:32,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:33,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:33,686 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:34,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:35,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:36,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:37,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:38,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:39,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:39,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:40,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:41,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:42,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:43,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:44,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:44,630 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:45,238 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:38:45,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:45,571 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:45,630 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:46,640 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:47,130 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:38:47,130 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 with policy now +2025-01-04 22:38:47,133 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:38:47,133 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:38:47,133 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:38:47,134 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:38:47,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:38:47,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:38:47,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:38:47,377 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/b5k7jxeb-media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:38:47,610 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/kf8cwkq9-media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:38:47,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:48,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:48,633 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:49,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:50,154 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:50,636 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:51,635 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:52,210 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:52,634 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:53,633 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:54,639 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:55,493 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:55,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:56,211 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:56,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:57,641 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:58,640 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:59,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:00,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:00,577 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:00,630 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:00,637 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:01,639 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:02,641 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:03,645 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:04,648 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:05,652 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:06,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:06,654 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:07,215 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:07,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:07,914 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:07,914 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:07,915 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:08,215 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:08,215 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:08,656 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:09,655 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:10,216 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:10,663 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:11,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:11,961 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:12,217 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:12,657 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:13,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:14,218 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:14,657 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:15,238 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:39:15,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:15,573 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:15,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:16,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:17,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:17,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:18,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:19,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:20,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:21,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:22,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:23,581 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:23,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:24,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:25,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:26,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:27,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:28,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:29,581 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:29,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:30,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:30,574 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:30,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:31,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:32,663 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:33,661 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:34,227 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:34,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:34,661 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:35,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:36,226 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:36,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:37,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:38,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:39,665 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:39,766 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:40,227 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:40,322 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:39:40,322 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 with policy now +2025-01-04 22:39:40,326 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:39:40,326 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:40,326 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:40,327 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:40,565 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/nqty559b-media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:39:40,675 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:41,227 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:41,228 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:39:41,228 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:39:41,493 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/aq7dgfx0-media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:39:41,681 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:42,228 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:42,679 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:43,683 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:44,678 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:45,238 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:39:45,239 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:45,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:45,583 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:45,686 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:46,229 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:46,689 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:47,687 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:48,230 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:48,691 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:48,964 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:48,965 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:48,965 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,077 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:39:49,077 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:39:49,229 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:39:49,232 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table +2025-01-04 22:39:49,447 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/6nyz0uda-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:39:49,461 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/fjs71n1j-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:49,519 INFO wandb-upload_2:2039437 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq64c5crp +2025-01-04 22:39:49,624 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:39:49,633 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:39:49,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:49,841 INFO SenderThread:2039437 [sender.py:send_request_log_artifact():1525] logged artifact run-amw07z8o-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5OTY1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NDA2OQ==', 'latestArtifact': None}} +2025-01-04 22:39:49,841 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:39:49,841 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-04 22:39:49,841 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:49,842 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,842 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,842 DEBUG SenderThread:2039437 [sender.py:send():379] send: telemetry +2025-01-04 22:39:49,842 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,846 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,846 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,847 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,847 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,847 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,847 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,848 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,848 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,848 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,848 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,849 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,849 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,849 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,849 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,850 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,850 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,850 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,850 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,851 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,851 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,851 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,851 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:39:50,056 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/r10vtkdr-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:39:50,178 INFO wandb-upload_2:2039437 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpidta_1x3 +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval +2025-01-04 22:39:50,589 INFO SenderThread:2039437 [sender.py:send_request_log_artifact():1525] logged artifact run-amw07z8o-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5OTY2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NDA3Mw==', 'latestArtifact': None}} +2025-01-04 22:39:50,589 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:39:50,589 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json with policy now +2025-01-04 22:39:50,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:50,591 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:50,591 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:50,592 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:50,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:50,757 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/jhan0803-media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:39:50,989 INFO wandb-upload_2:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/dk0m0ord-media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:39:51,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:51,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:52,234 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:52,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:53,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:54,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:55,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:56,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:56,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:57,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:58,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:58,737 DEBUG SenderThread:2039437 [sender.py:send():379] send: exit +2025-01-04 22:39:58,737 INFO SenderThread:2039437 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:39:58,737 INFO SenderThread:2039437 [sender.py:send_exit():588] handling runtime: 283 +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:send_exit():594] send defer +2025-01-04 22:39:58,739 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,739 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:39:58,739 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:39:58,739 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,739 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:39:58,739 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,740 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:39:58,740 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:39:58,740 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,740 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:39:58,740 INFO HandlerThread:2039437 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:39:58,740 DEBUG SystemMonitor:2039437 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:39:58,740 DEBUG SystemMonitor:2039437 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:39:58,740 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:39:58,741 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:39:58,792 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:39:58,792 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:39:58,792 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:39:58,792 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,792 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:39:58,793 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:39:58,793 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:39:58,793 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,793 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:39:58,793 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,793 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:39:58,793 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:39:58,793 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,793 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:39:59,237 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:40:00,237 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:40:04,971 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:04,971 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:04,971 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:40:04,972 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:04,972 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:40:04,972 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:40:04,972 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:04,972 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:40:04,973 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:40:04,973 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:04,973 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:40:04,973 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:40:04,974 DEBUG SenderThread:2039437 [sender.py:send():379] send: summary +2025-01-04 22:40:04,977 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:40:04,978 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:04,978 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:40:04,978 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:40:04,978 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:04,978 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:40:04,978 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:04,979 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:40:04,983 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:05,079 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:40:05,080 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:05,080 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:40:05,080 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:05,080 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:40:05,239 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:40:05,239 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml +2025-01-04 22:40:05,738 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:07,749 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:40:07,749 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:07,749 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:07,749 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:40:07,749 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:07,750 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:40:07,750 INFO SenderThread:2039437 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:40:07,752 INFO SenderThread:2039437 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:40:07,754 INFO SenderThread:2039437 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:40:07,766 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:40:07,766 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:07,766 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:40:07,766 DEBUG SenderThread:2039437 [sender.py:send():379] send: artifact +2025-01-04 22:40:07,976 INFO SenderThread:2039437 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:40:07,976 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:07,976 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:40:07,976 INFO SenderThread:2039437 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:40:08,240 INFO SenderThread:2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml config.yaml +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json wandb-summary.json +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt requirements.txt +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log output.log +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch diff.patch +2025-01-04 22:40:08,242 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:40:08,247 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:40:08,248 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,248 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:40:08,252 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,253 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:40:08,253 INFO SenderThread:2039437 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:40:08,424 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:40:08,436 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml +2025-01-04 22:40:08,464 INFO wandb-upload_2:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/cbx85sls-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:40:08,468 INFO wandb-upload_4:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:40:08,471 INFO wandb-upload_3:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt +2025-01-04 22:40:08,672 INFO Thread-11 (_thread_body):2039437 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:40:08,672 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,672 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:40:08,672 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,672 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:40:08,672 INFO SenderThread:2039437 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:40:08,674 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:40:08,674 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,674 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:40:08,674 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,674 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:40:08,674 INFO SenderThread:2039437 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:40:08,740 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:08,873 INFO SenderThread:2039437 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:40:08,873 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:40:08,873 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:08,873 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,873 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:40:08,873 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,873 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:40:08,874 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:40:08,874 DEBUG SenderThread:2039437 [sender.py:send():379] send: final +2025-01-04 22:40:08,874 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,874 DEBUG SenderThread:2039437 [sender.py:send():379] send: footer +2025-01-04 22:40:08,874 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:40:08,874 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,874 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:40:08,875 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:08,875 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:08,875 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:40:08,875 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:08,875 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:40:08,877 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:40:08,877 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:40:08,878 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:08,938 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:08,938 INFO MainThread:2039437 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:40:08,938 INFO MainThread:2039437 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:40:08,938 INFO MainThread:2039437 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:40:08,939 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:40:08,939 INFO HandlerThread:2039437 [handler.py:finish():882] shutting down handler +2025-01-04 22:40:09,875 INFO WriterThread:2039437 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb +2025-01-04 22:40:09,938 INFO SenderThread:2039437 [sender.py:finish():1615] shutting down sender +2025-01-04 22:40:09,938 INFO SenderThread:2039437 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:40:09,938 INFO SenderThread:2039437 [file_pusher.py:join():175] waiting for file pusher diff --git a/amw07z8o/wandb/debug.log b/amw07z8o/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cc6049202b02f3ac61ab96f41a9caf6588c6298f --- /dev/null +++ b/amw07z8o/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Configure stats pid to 2039256 +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/settings +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug.log +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug-internal.log +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o'} +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():619] starting backend +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():623] setting up manager +2025-01-04 22:35:14,864 INFO MainThread:2039256 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:35:14,865 INFO MainThread:2039256 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:35:14,881 INFO MainThread:2039256 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:35:14,887 INFO MainThread:2039256 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:35:15,152 INFO MainThread:2039256 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:35:15,221 INFO MainThread:2039256 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:35:15,221 INFO MainThread:2039256 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:35:15,571 INFO MainThread:2039256 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:35:15,571 INFO MainThread:2039256 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/tb, True +2025-01-04 22:35:22,428 INFO MainThread:2039256 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:40:09,941 WARNING MsgRouterThr:2039256 [router.py:message_loop():77] message_loop has been closed diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..897b993fbc596d264a44c1bc6f72996fac48bb2c --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736048114 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c30a13305f74325cdb6b6fee6f57b4b907700d90 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.12855625160016762, 0.9510095416597523, 0.1475618631352536, 0.9175734370111737, 0.25050920253021436, 0.8823125510643227, 0.3999994695386616, 0.8205093293741077]]} \ No newline at end of file diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9f50d6c776ced77c4fec8ef3d2c625575ef10e35 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[51, 55, 0.9272727272727272, 0.8096788226938778, 0.9825406036305941, 0.8389098643578772, 0.97951644276923, 0.8550868853216484, 0.9686893056140744, 0.8636523372697704, 0.9668117732738638]]} \ No newline at end of file diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4785ad9530c173cf6a520c308069fb377c94e9c0 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17dfc8269beff249beeaa68babd76191e5205d080336e6c7317121a355b58d2e +size 1201547 diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92a134ce6a2c2a0894be481c99da0bd6b2803058 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b4da146e47f9bb9dd381e6c5e125d416e0fca2c5a980fa101eaa6445138b807 +size 1197063 diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ac6df58df8aeafba688205104d851efd9e48509 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a498ba0c97696b225f3555a898d21e754da3884a554f9a87bddfe6e7c8125c15 +size 1211751 diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78638d5060651ade632be3d40cdbcde8f74dbcb3 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b597141eccbb7769cb09cb50fb9673419331351d8fa33477f4377d2ed804a388 +size 1195007 diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..287832894886a538330c9b9f3afdefff1a259734 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1da214e6639c86f006cb2279112de21986fbe0aed182713e0621d957827c9544 +size 1241031 diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6eb4af93ddd1eb7da2f16ca69a62745dc3ff5e27 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736048122.429481]: resetting environment +[INFO] [1736048122.436558]: cleared current plan +[INFO] [1736048122.437574]: moving to home +[INFO] [1736048123.440345]: reached home +[INFO] [1736048123.441598]: reset action history +[INFO] [1736048126.150691]: environment reset +Starting episode 0 +[INFO] [1736048126.152417]: resetting environment +[INFO] [1736048126.154955]: cleared current plan +[INFO] [1736048126.155718]: moving to home +[INFO] [1736048127.157272]: reached home +[INFO] [1736048127.158973]: reset action history +[INFO] [1736048129.867994]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736048133.004753]: resetting environment +[INFO] [1736048133.008335]: cleared current plan +[INFO] [1736048133.009437]: moving to home +[INFO] [1736048134.011791]: reached home +[INFO] [1736048134.013312]: reset action history +[INFO] [1736048136.722057]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/episode_rosbags/episode_0_2025-01-04-22-35-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2039256.log +started roslaunch server http://158.130.50.37:44727/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2039540] +process[print_text-2]: started with pid [2039552] +process[rosbag_record-3]: started with pid [2039576] +started bagging! +For topic gripper_width: timestamp difference is 124629672 for nearest: 1736048149018175061 - target: 1736048148893545389 at idx 157 +[INFO] [1736048163.741819]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1736048164.463433]: Stored episode 1. +[INFO] [1736048164.465402]: Saving video... +[INFO] [1736048164.670516]: Video saved! +Starting episode 1 +[INFO] [1736048164.682475]: resetting environment +[INFO] [1736048164.688591]: cleared current plan +[INFO] [1736048164.689547]: moving to home +[INFO] [1736048168.191873]: reached home +[INFO] [1736048168.193079]: reset action history +[INFO] [1736048170.900939]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048183.888829]: resetting environment +[INFO] [1736048183.893063]: cleared current plan +[INFO] [1736048183.893562]: moving to home +[INFO] [1736048184.895082]: reached home +[INFO] [1736048184.895494]: reset action history +[INFO] [1736048187.601958]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/episode_rosbags/episode_1_2025-01-04-22-36-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2039256.log +started roslaunch server http://158.130.50.37:36063/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2039687] +process[print_text-5]: started with pid [2039688] +process[rosbag_record-6]: started with pid [2039712] +started bagging! +[INFO] [1736048214.564942]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736048215.351858]: Stored episode 2. +[INFO] [1736048215.354785]: Saving video... +[INFO] [1736048215.676723]: Video saved! +Starting episode 2 +[INFO] [1736048215.707128]: resetting environment +[INFO] [1736048215.712922]: cleared current plan +[INFO] [1736048215.713671]: moving to home +[INFO] [1736048219.216828]: reached home +[INFO] [1736048219.219887]: reset action history +[INFO] [1736048221.933926]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048223.565190]: resetting environment +[INFO] [1736048223.577822]: cleared current plan +[INFO] [1736048223.578711]: moving to home +[INFO] [1736048224.580283]: reached home +[INFO] [1736048224.586014]: reset action history +[INFO] [1736048227.294517]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/episode_rosbags/episode_2_2025-01-04-22-37-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2039256.log +started roslaunch server http://158.130.50.37:40197/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2039833] +process[print_text-8]: started with pid [2039834] +process[rosbag_record-9]: started with pid [2039858] +started bagging! +For topic gripper_width: timestamp difference is 104746974 for nearest: 1736048248118177093 - target: 1736048248013430119 at idx 249 +[INFO] [1736048254.265961]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736048255.021743]: Stored episode 3. +[INFO] [1736048255.022868]: Saving video... +[WARN] [1736048255.607034]: Plan exhausted +[WARN] [1736048255.647061]: Plan exhausted +[WARN] [1736048255.689305]: Plan exhausted +[WARN] [1736048255.728267]: Plan exhausted +[WARN] [1736048255.767137]: Plan exhausted +[INFO] [1736048255.701993]: Video saved! +Starting episode 3 +[INFO] [1736048255.760742]: resetting environment +[INFO] [1736048255.772699]: cleared current plan +[INFO] [1736048255.776417]: moving to home +[INFO] [1736048258.983330]: reached home +[INFO] [1736048258.986729]: reset action history +[INFO] [1736048261.725393]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048294.978094]: resetting environment +[INFO] [1736048294.992418]: cleared current plan +[INFO] [1736048294.992643]: moving to home +[INFO] [1736048295.994438]: reached home +[INFO] [1736048295.998108]: reset action history +[INFO] [1736048298.871623]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/episode_rosbags/episode_3_2025-01-04-22-38-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2039256.log +started roslaunch server http://158.130.50.37:33405/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2040044] +process[print_text-11]: started with pid [2040045] +process[rosbag_record-12]: started with pid [2040046] +started bagging! +[WARN] [1736048327.030874]: Plan exhausted +[WARN] [1736048327.057698]: Plan exhausted +[WARN] [1736048327.097712]: Plan exhausted +[WARN] [1736048327.137884]: Plan exhausted +[INFO] [1736048325.831522]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736048326.608752]: Stored episode 4. +[INFO] [1736048326.611985]: Saving video... +[INFO] [1736048327.094677]: Video saved! +Starting episode 4 +[INFO] [1736048327.134168]: resetting environment +[INFO] [1736048327.148845]: cleared current plan +[INFO] [1736048327.152822]: moving to home +[INFO] [1736048330.658891]: reached home +[INFO] [1736048330.663295]: reset action history +[INFO] [1736048333.446177]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048347.140742]: resetting environment +[INFO] [1736048347.167087]: cleared current plan +[INFO] [1736048347.169845]: moving to home +[INFO] [1736048348.173351]: reached home +[INFO] [1736048348.182895]: reset action history +[INFO] [1736048350.916363]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/episode_rosbags/episode_4_2025-01-04-22-39-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2039256.log +started roslaunch server http://158.130.50.37:41789/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2040205] +process[print_text-14]: started with pid [2040206] +process[rosbag_record-15]: started with pid [2040207] +started bagging! +For topic gripper_width: timestamp difference is 103871968 for nearest: 1736048373418175128 - target: 1736048373314303160 at idx 249 +For topic gripper_width: timestamp difference is 104490856 for nearest: 1736048373618173698 - target: 1736048373513682842 at idx 249 +For topic gripper_width: timestamp difference is 107778851 for nearest: 1736048377818175379 - target: 1736048377710396528 at idx 249 +[INFO] [1736048379.008950]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736048379.765636]: Stored episode 5. +[INFO] [1736048379.770463]: Saving video... +[INFO] [1736048380.259675]: Video saved! +[INFO] [1736048380.326441]: resetting environment +[INFO] [1736048380.357930]: cleared current plan +[INFO] [1736048380.361522]: moving to home +[INFO] [1736048383.576455]: reached home +[INFO] [1736048383.589139]: reset action history +[INFO] [1736048386.450864]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..261822b808e7d260e64fd229d884f66390fbca7e --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:35:15.245061", + "startedAt": "2025-01-05T03:35:14.853349", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.794625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.037, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 774.6898498535156 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..47e3d866babe58e082d50644a44b1c05c248e027 --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "17dfc8269beff249beeaa68babd76191e5205d080336e6c7317121a355b58d2e", "size": 1201547, "path": "media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4"}, "global_step": 664, "_timestamp": 1736048389.5332634, "_runtime": 274.66797852516174, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6b4da146e47f9bb9dd381e6c5e125d416e0fca2c5a980fa101eaa6445138b807", "size": 1197063, "path": "media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4"}, "eval/num_success": 3.0, "episode": 55.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.1285562515258789, "eval/success_rate_ub_0.01": 0.9510095119476318, "eval/success_rate_lb_0.025": 0.14756186306476593, "eval/success_rate_ub_0.025": 0.9175734519958496, "eval/success_rate_lb_0.05": 0.25050920248031616, "eval/success_rate_ub_0.05": 0.882312536239624, "eval/success_rate_lb_0.1": 0.3999994695186615, "eval/success_rate_ub_0.1": 0.8205093145370483, "eval/2_eval": {"_type": "video-file", "sha256": "a498ba0c97696b225f3555a898d21e754da3884a554f9a87bddfe6e7c8125c15", "size": 1211751, "path": "media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b597141eccbb7769cb09cb50fb9673419331351d8fa33477f4377d2ed804a388", "size": 1195007, "path": "media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1da214e6639c86f006cb2279112de21986fbe0aed182713e0621d957827c9544", "size": 1241031, "path": "media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "bb2b66ada2f22e77fd7f6070fb35ca74f8550a1117f4b1b40bac2837d171cf7c", "size": 400, "artifact_path": "wandb-client-artifact://ghiyf094s3wkxs17v7x9fqh76hybrhembf0nwsexsdk2fw8yym5sohhcdch2p14lsm7ntj3qy3wfhjkuy6pfnquf1j5sqoh1um2w6pv70xwzgigahupg3tirdpvxucfp/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://j268ypgytaw27i8fa1wee120y4l8d9zc7p7tublqsxxzz7ruvwj732mwj3e5jd6aiokrwf964d3mq8rtcxdis7uyanyyppz3klp5emmz7j7409b9yrnzerqpqluo1mfz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 51, "total_num_episodes": 55, "total_success_rate": 0.9272727272727272, "total_success_rate_lb_0.01": 0.8096788226938778, "total_success_rate_ub_0.01": 0.9825406036305941, "total_success_rate_lb_0.025": 0.8389098643578772, "total_success_rate_ub_0.025": 0.97951644276923, "total_success_rate_lb_0.05": 0.8550868853216484, "total_success_rate_ub_0.05": 0.9686893056140744, "total_success_rate_lb_0.1": 0.8636523372697704, "total_success_rate_ub_0.1": 0.9668117732738638, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c50ce8d51f945b9ec47955de369e3979c7775913743278d5e4313e500349217a", "size": 518, "artifact_path": "wandb-client-artifact://9drqeuc3lcexkxbygd23wl3x4ik6qc7fx75dzsvog7v1d30t9fij6x51cuvuyd6qbkdwk4l9n422dau4i3sj1k06hb7xffnb91bbf4k1udqpk9ltoonqakt6h46xua31/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://kgarfeqo01fhfutf9mk0g047zs7kq6q1p37blg0ukohqbvz202vppv7xdvbjra6qm8a1lzhtslz7crk9ry01uztk0eh5rl10zrw57ns0z57qpmoc4n9os297s52k7199:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 283}, "eval/total_success_rate": 0.9272727370262146, "eval/total_success_rate_lb_0.01": 0.8096787929534912, "eval/total_success_rate_ub_0.01": 0.9825406074523926, "eval/total_success_rate_lb_0.025": 0.8389098644256592, "eval/total_success_rate_ub_0.025": 0.9795164465904236, "eval/total_success_rate_lb_0.05": 0.8550868630409241, "eval/total_success_rate_ub_0.05": 0.9686893224716187, "eval/total_success_rate_lb_0.1": 0.8636523485183716, "eval/total_success_rate_ub_0.1": 0.9668117761611938} \ No newline at end of file diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug-internal.log b/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c30f10401ccb487f1a3a2dd6731e1b927b4c192a --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug-internal.log @@ -0,0 +1,790 @@ +2025-01-04 22:35:14,865 INFO StreamThr :2039437 [internal.py:wandb_internal():85] W&B internal server running at pid: 2039437, started at: 2025-01-04 22:35:14.864650 +2025-01-04 22:35:14,865 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:35:14,883 INFO WriterThread:2039437 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb +2025-01-04 22:35:14,883 DEBUG SenderThread:2039437 [sender.py:send():379] send: header +2025-01-04 22:35:14,889 DEBUG SenderThread:2039437 [sender.py:send():379] send: run +2025-01-04 22:35:15,137 INFO SenderThread:2039437 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files +2025-01-04 22:35:15,137 INFO SenderThread:2039437 [sender.py:_start_run_threads():1188] run started: amw07z8o with start time 1736048114.865285 +2025-01-04 22:35:15,152 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:35:15,153 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:35:15,225 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:35:15,230 DEBUG HandlerThread:2039437 [system_info.py:__init__():26] System info init +2025-01-04 22:35:15,231 DEBUG HandlerThread:2039437 [system_info.py:__init__():41] System info init done +2025-01-04 22:35:15,231 INFO HandlerThread:2039437 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:35:15,231 INFO SystemMonitor:2039437 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:35:15,231 INFO HandlerThread:2039437 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:35:15,232 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:35:15,232 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:35:15,233 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:35:15,233 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:35:15,234 INFO SystemMonitor:2039437 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:35:15,244 DEBUG HandlerThread:2039437 [system_info.py:probe():152] Probing system +2025-01-04 22:35:15,246 DEBUG HandlerThread:2039437 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_info.py:probe():200] Probing system done +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:35:15.245061', 'startedAt': '2025-01-05T03:35:14.853349', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.794625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.037, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 774.6898498535156}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:35:15,251 INFO HandlerThread:2039437 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:35:15,251 INFO HandlerThread:2039437 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:35:15,251 DEBUG HandlerThread:2039437 [system_info.py:_save_code():44] Saving code +2025-01-04 22:35:15,252 DEBUG HandlerThread:2039437 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:35:15,252 DEBUG HandlerThread:2039437 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:35:15,315 DEBUG HandlerThread:2039437 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:35:15,315 INFO HandlerThread:2039437 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:35:15,318 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:35:15,318 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:35:15,318 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:35:15,319 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:35:15,564 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/hca1n5xm-wandb-metadata.json +2025-01-04 22:35:15,566 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:35:15,566 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:35:15,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:35:15,567 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:35:15,568 INFO wandb-upload_2:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/p3bg4nbc-diff.patch +2025-01-04 22:35:15,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:15,571 INFO HandlerThread:2039437 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/tb" + save: true +} +control { + relay_id: "e3c9300cb5724fd08f0cdf614d01cd70" +} +_info { + stream_id: "amw07z8o" +} + +2025-01-04 22:35:15,572 DEBUG HandlerThread:2039437 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:35:15,597 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/8pqe5g39-code/FISH/eval_robot.py +2025-01-04 22:35:15,683 DEBUG SenderThread:2039437 [sender.py:send():379] send: telemetry +2025-01-04 22:35:15,683 DEBUG SenderThread:2039437 [sender.py:send():379] send: tbrecord +2025-01-04 22:35:15,684 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:35:15,684 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 with policy live +2025-01-04 22:35:15,859 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/8dz9soy1-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:35:16,139 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:35:16,139 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt +2025-01-04 22:35:16,139 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH +2025-01-04 22:35:16,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code +2025-01-04 22:35:16,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:17,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:18,138 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:18,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:19,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:20,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:20,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:21,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:22,429 DEBUG SenderThread:2039437 [sender.py:send():379] send: config +2025-01-04 22:35:22,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:23,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:24,140 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:24,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:25,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:26,141 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:26,151 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:26,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:27,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:28,141 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:28,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:29,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:30,142 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:30,566 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:35:30,567 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:35:30,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:31,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:31,666 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:32,142 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:32,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:33,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:34,143 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:34,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:35,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:36,144 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:36,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:36,722 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:37,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:38,144 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:38,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:39,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:40,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:41,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:42,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:42,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:43,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:44,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:45,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:35:45,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:35:45,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:46,572 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:47,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:47,720 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:48,147 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml +2025-01-04 22:35:48,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:49,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:50,148 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:35:50,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:51,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:52,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:53,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:53,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:54,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:55,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:56,574 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:57,575 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:58,575 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:35:59,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:35:59,575 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:00,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:00,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:00,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:01,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:02,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:03,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:04,579 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:04,671 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:04,713 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:36:04,714 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:36:04,714 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 with policy now +2025-01-04 22:36:04,714 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:36:04,714 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:36:04,714 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:36:04,961 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/sdieoeko-media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:36:05,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos +2025-01-04 22:36:05,411 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/k1hw2xsb-media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:36:05,581 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:06,152 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:06,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:07,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:08,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:09,582 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:10,153 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:10,237 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:10,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:11,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:12,154 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:12,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:13,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:14,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:15,234 DEBUG SystemMonitor:2039437 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:36:15,235 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:36:15,567 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:15,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:15,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:15,663 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:16,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:17,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:18,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:19,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:20,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:21,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:21,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:22,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:23,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:24,158 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:36:24,390 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/kpdx4oln-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:36:24,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:25,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:26,159 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:26,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:26,585 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:27,585 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:28,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:29,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:30,160 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:30,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:30,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:30,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:31,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:31,624 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:32,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:33,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:34,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:35,586 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:36,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:37,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:37,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:38,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:39,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:40,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:41,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:42,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:43,572 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:43,587 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:44,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:45,236 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:36:45,568 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:36:45,568 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:36:45,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:46,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:47,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:48,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:48,651 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:49,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:50,588 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:51,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:52,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:53,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:54,565 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:36:54,605 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:55,591 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:55,721 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:36:55,730 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:36:55,730 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 with policy now +2025-01-04 22:36:55,730 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:36:55,731 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:36:55,731 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:36:55,987 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/r6xevbwr-media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:36:56,168 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:36:56,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:56,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:36:56,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:36:56,423 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/gfouxrg2-media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:36:56,595 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:57,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:58,169 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:36:58,592 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:36:59,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:00,170 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:00,261 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:00,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:00,570 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:00,617 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:01,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:02,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:03,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:03,980 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:37:03,980 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:37:03,985 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:37:04,171 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:37:04,171 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:37:04,171 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:04,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:05,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:05,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:06,172 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:06,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:07,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:08,173 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:08,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:09,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:10,173 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:10,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:11,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:11,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:12,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:13,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:14,593 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:15,236 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:37:15,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:15,569 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:15,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:16,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:16,721 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:17,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:18,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:19,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:20,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:21,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:22,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:22,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:23,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:24,594 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:25,595 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:26,595 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:27,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:27,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:28,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:29,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:30,181 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:30,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:30,569 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:30,617 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:31,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:32,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:32,667 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:33,596 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:34,597 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:35,598 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:35,751 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:37:35,751 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 with policy now +2025-01-04 22:37:35,754 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:37:35,755 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:37:35,755 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:37:35,756 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:37:36,000 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/csni4vgu-media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:37:36,183 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:37:36,431 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/yogqmswq-media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:37:36,616 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:37,600 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:37,777 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:38,184 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:38,605 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:39,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:40,184 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:40,609 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:41,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:42,600 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:43,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:43,769 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:44,186 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:37:44,602 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:45,237 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:37:45,569 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:37:45,570 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:37:45,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:46,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:47,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:48,608 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:49,611 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:49,683 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:50,606 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:51,610 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:52,611 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:53,611 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:54,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:55,579 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:37:55,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:56,612 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:57,612 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:58,617 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:37:59,614 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:00,570 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:00,570 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:00,613 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:00,655 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:01,614 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:02,615 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:03,616 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:04,619 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:05,620 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:06,579 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:06,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:07,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:08,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:09,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:10,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:11,585 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:11,621 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:12,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:13,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:14,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:15,196 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:38:15,237 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:38:15,424 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/ydkdtee9-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:38:15,572 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:15,573 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:15,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:15,850 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:38:15,850 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:38:15,850 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:38:16,196 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:38:16,196 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:16,624 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:17,037 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:17,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:18,197 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:18,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:19,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:20,198 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:20,625 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:21,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:22,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:22,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:23,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:24,626 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:25,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:26,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:27,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:28,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:28,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:29,627 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:30,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:30,571 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:30,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:31,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:32,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:33,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:33,686 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:34,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:35,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:36,628 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:37,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:38,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:39,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:39,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:40,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:41,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:42,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:43,629 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:44,578 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:44,630 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:45,238 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:38:45,571 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:38:45,571 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:38:45,630 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:46,640 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:47,130 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:38:47,130 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 with policy now +2025-01-04 22:38:47,133 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:38:47,133 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:38:47,133 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:38:47,134 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:38:47,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:38:47,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:38:47,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:38:47,377 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/b5k7jxeb-media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:38:47,610 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/kf8cwkq9-media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:38:47,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:48,208 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:48,633 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:49,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:50,154 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:50,636 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:51,635 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:52,210 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:52,634 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:53,633 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:54,639 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:55,493 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:38:55,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:56,211 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:38:56,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:57,641 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:58,640 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:38:59,638 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:00,577 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:00,577 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:00,630 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:00,637 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:01,639 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:02,641 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:03,645 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:04,648 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:05,652 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:06,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:06,654 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:07,215 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:07,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:07,914 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:07,914 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:07,915 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:08,215 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:08,215 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:08,656 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:09,655 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:10,216 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:10,663 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:11,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:11,961 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:12,217 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:12,657 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:13,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:14,218 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:14,657 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:15,238 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:39:15,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:15,573 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:15,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:16,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:17,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:17,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:18,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:19,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:20,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:21,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:22,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:23,581 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:23,658 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:24,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:25,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:26,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:27,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:28,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:29,581 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:29,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:30,573 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:30,574 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:30,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:31,659 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:32,663 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:33,661 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:34,227 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:34,623 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:34,661 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:35,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:36,226 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:36,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:37,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:38,662 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:39,665 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:39,766 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:40,227 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:40,322 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:39:40,322 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 with policy now +2025-01-04 22:39:40,326 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:39:40,326 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:40,326 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:40,327 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:40,565 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/nqty559b-media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:39:40,675 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:41,227 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:41,228 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:39:41,228 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval +2025-01-04 22:39:41,493 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/aq7dgfx0-media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:39:41,681 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:42,228 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:42,679 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:43,683 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:44,678 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:45,238 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:39:45,239 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:45,583 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:39:45,583 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:39:45,686 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:46,229 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:46,689 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:47,687 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:48,230 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:48,691 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:48,964 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:48,965 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:48,965 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,077 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:39:49,077 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:39:49,229 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:39:49,232 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval +2025-01-04 22:39:49,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table +2025-01-04 22:39:49,447 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/6nyz0uda-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:39:49,461 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/fjs71n1j-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:49,519 INFO wandb-upload_2:2039437 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq64c5crp +2025-01-04 22:39:49,624 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:39:49,633 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:39:49,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:49,841 INFO SenderThread:2039437 [sender.py:send_request_log_artifact():1525] logged artifact run-amw07z8o-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5OTY1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NDA2OQ==', 'latestArtifact': None}} +2025-01-04 22:39:49,841 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:39:49,841 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-04 22:39:49,841 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:49,842 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,842 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,842 DEBUG SenderThread:2039437 [sender.py:send():379] send: telemetry +2025-01-04 22:39:49,842 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,846 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,846 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,847 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,847 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,847 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,847 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,848 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,848 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,848 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,848 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,849 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,849 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,849 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,849 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,850 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,850 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,850 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,850 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,851 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,851 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:49,851 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:49,851 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:39:50,056 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/r10vtkdr-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:39:50,178 INFO wandb-upload_2:2039437 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpidta_1x3 +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:39:50,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval +2025-01-04 22:39:50,589 INFO SenderThread:2039437 [sender.py:send_request_log_artifact():1525] logged artifact run-amw07z8o-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5OTY2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NDA3Mw==', 'latestArtifact': None}} +2025-01-04 22:39:50,589 DEBUG SenderThread:2039437 [sender.py:send():379] send: files +2025-01-04 22:39:50,589 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json with policy now +2025-01-04 22:39:50,589 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:50,591 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:39:50,591 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:39:50,592 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:50,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:50,757 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/jhan0803-media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:39:50,989 INFO wandb-upload_2:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/dk0m0ord-media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:39:51,233 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:39:51,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:52,234 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:39:52,684 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:53,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:54,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:55,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:56,584 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:39:56,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:57,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:58,685 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:39:58,737 DEBUG SenderThread:2039437 [sender.py:send():379] send: exit +2025-01-04 22:39:58,737 INFO SenderThread:2039437 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:39:58,737 INFO SenderThread:2039437 [sender.py:send_exit():588] handling runtime: 283 +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:send_exit():594] send defer +2025-01-04 22:39:58,739 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,739 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:39:58,739 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:39:58,739 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:39:58,739 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,739 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:39:58,739 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,740 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:39:58,740 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:39:58,740 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,740 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:39:58,740 INFO HandlerThread:2039437 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:39:58,740 DEBUG SystemMonitor:2039437 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:39:58,740 DEBUG SystemMonitor:2039437 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:39:58,740 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:39:58,741 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:39:58,792 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:39:58,792 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:39:58,792 INFO HandlerThread:2039437 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:39:58,792 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,792 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:39:58,793 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:39:58,793 DEBUG SenderThread:2039437 [sender.py:send():379] send: stats +2025-01-04 22:39:58,793 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,793 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:39:58,793 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:39:58,793 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:39:58,793 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:39:58,793 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:39:58,793 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:39:59,237 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:40:00,237 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:40:04,971 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:04,971 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:04,971 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:40:04,972 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:04,972 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:40:04,972 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:40:04,972 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:04,972 DEBUG SenderThread:2039437 [sender.py:send():379] send: history +2025-01-04 22:40:04,973 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:40:04,973 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:04,973 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:40:04,973 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:40:04,974 DEBUG SenderThread:2039437 [sender.py:send():379] send: summary +2025-01-04 22:40:04,977 INFO SenderThread:2039437 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:40:04,978 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:04,978 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:40:04,978 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:40:04,978 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:04,978 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:40:04,978 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:04,979 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:40:04,983 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:05,079 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:40:05,080 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:05,080 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:40:05,080 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:05,080 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:40:05,239 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:40:05,239 INFO Thread-12 :2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml +2025-01-04 22:40:05,738 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:07,749 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:40:07,749 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:07,749 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:07,749 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:40:07,749 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:07,750 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:40:07,750 INFO SenderThread:2039437 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:40:07,752 INFO SenderThread:2039437 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:40:07,754 INFO SenderThread:2039437 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:40:07,766 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:40:07,766 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:07,766 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:40:07,766 DEBUG SenderThread:2039437 [sender.py:send():379] send: artifact +2025-01-04 22:40:07,976 INFO SenderThread:2039437 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:40:07,976 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:07,976 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:40:07,976 INFO SenderThread:2039437 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:40:08,240 INFO SenderThread:2039437 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml config.yaml +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json wandb-summary.json +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt requirements.txt +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log output.log +2025-01-04 22:40:08,241 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/diff.patch diff.patch +2025-01-04 22:40:08,242 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 media/videos/eval/3_eval_3_b597141eccbb7769cb09.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 media/videos/eval/2_eval_2_a498ba0c97696b225f35.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 media/videos/eval/4_eval_4_1da214e6639c86f006cb.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 media/videos/eval/0_eval_0_17dfc8269beff249beea.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 media/videos/eval/1_eval_1_6b4da146e47f9bb9dd38.mp4 +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json media/table/eval/total_success_rate_ci_6_c50ce8d51f945b9ec479.table.json +2025-01-04 22:40:08,246 INFO SenderThread:2039437 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-04 22:40:08,247 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:40:08,248 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,248 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:40:08,252 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,253 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:40:08,253 INFO SenderThread:2039437 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:40:08,424 INFO wandb-upload_0:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/wandb-summary.json +2025-01-04 22:40:08,436 INFO wandb-upload_1:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/config.yaml +2025-01-04 22:40:08,464 INFO wandb-upload_2:2039437 [upload_job.py:push():130] Uploaded file /tmp/tmpvjnjcsqwwandb/cbx85sls-events.out.tfevents.1736048115.leonmkim-ROG-Strix-G15CS-G15CS.2039256.0 +2025-01-04 22:40:08,468 INFO wandb-upload_4:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/output.log +2025-01-04 22:40:08,471 INFO wandb-upload_3:2039437 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/files/requirements.txt +2025-01-04 22:40:08,672 INFO Thread-11 (_thread_body):2039437 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:40:08,672 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,672 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:40:08,672 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,672 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:40:08,672 INFO SenderThread:2039437 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:40:08,674 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:40:08,674 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,674 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:40:08,674 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,674 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:40:08,674 INFO SenderThread:2039437 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:40:08,740 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:08,873 INFO SenderThread:2039437 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:40:08,873 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:40:08,873 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:08,873 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,873 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:40:08,873 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,873 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:40:08,874 INFO SenderThread:2039437 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:40:08,874 DEBUG SenderThread:2039437 [sender.py:send():379] send: final +2025-01-04 22:40:08,874 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:40:08,874 DEBUG SenderThread:2039437 [sender.py:send():379] send: footer +2025-01-04 22:40:08,874 INFO HandlerThread:2039437 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:40:08,874 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: defer +2025-01-04 22:40:08,874 INFO SenderThread:2039437 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:40:08,875 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:08,875 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:08,875 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:40:08,875 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:40:08,875 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:40:08,877 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:40:08,877 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:40:08,878 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:08,938 DEBUG SenderThread:2039437 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:40:08,938 INFO MainThread:2039437 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:40:08,938 INFO MainThread:2039437 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:40:08,938 INFO MainThread:2039437 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:40:08,939 DEBUG HandlerThread:2039437 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:40:08,939 INFO HandlerThread:2039437 [handler.py:finish():882] shutting down handler +2025-01-04 22:40:09,875 INFO WriterThread:2039437 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb +2025-01-04 22:40:09,938 INFO SenderThread:2039437 [sender.py:finish():1615] shutting down sender +2025-01-04 22:40:09,938 INFO SenderThread:2039437 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:40:09,938 INFO SenderThread:2039437 [file_pusher.py:join():175] waiting for file pusher diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug.log b/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cc6049202b02f3ac61ab96f41a9caf6588c6298f --- /dev/null +++ b/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Configure stats pid to 2039256 +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/settings +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:35:14,859 INFO MainThread:2039256 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug.log +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/wandb/run-20250104_223514-amw07z8o/logs/debug-internal.log +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o'} +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():619] starting backend +2025-01-04 22:35:14,860 INFO MainThread:2039256 [wandb_init.py:init():623] setting up manager +2025-01-04 22:35:14,864 INFO MainThread:2039256 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:35:14,865 INFO MainThread:2039256 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:35:14,881 INFO MainThread:2039256 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:35:14,887 INFO MainThread:2039256 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:35:15,152 INFO MainThread:2039256 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:35:15,221 INFO MainThread:2039256 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:35:15,221 INFO MainThread:2039256 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:35:15,567 INFO MainThread:2039256 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:35:15,571 INFO MainThread:2039256 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:35:15,571 INFO MainThread:2039256 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/amw07z8o/tb, True +2025-01-04 22:35:22,428 INFO MainThread:2039256 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:40:09,941 WARNING MsgRouterThr:2039256 [router.py:message_loop():77] message_loop has been closed diff --git a/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb b/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb new file mode 100644 index 0000000000000000000000000000000000000000..29a55da2347a09f2b3e5c33c6fe34e1fa4edf914 Binary files /dev/null and b/amw07z8o/wandb/run-20250104_223514-amw07z8o/run-amw07z8o.wandb differ diff --git a/irknmhae/.hydra/config.yaml b/irknmhae/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/irknmhae/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/irknmhae/.hydra/hydra.yaml b/irknmhae/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb121b082644c7c837cce2875b3455c31031cebb --- /dev/null +++ b/irknmhae/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/irknmhae + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/irknmhae/.hydra/overrides.yaml b/irknmhae/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/irknmhae/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/irknmhae/eval_policy.log b/irknmhae/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..dc04dff088129b38b739e3fd86ba8140847ba602 --- /dev/null +++ b/irknmhae/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 22:39:29,543][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:39:29,546][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 22:39:33,041][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/n627ep2q/episode_rosbags/episode_2_2025-01-03-22-42-16.bag b/n627ep2q/episode_rosbags/episode_2_2025-01-03-22-42-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..8dd3de05943389986cd9366e5eeaf7ae359a8b52 --- /dev/null +++ b/n627ep2q/episode_rosbags/episode_2_2025-01-03-22-42-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad9f3679fa63654782b4f1c5ccfb2e3a52d9b24b47df342e470d434bb9fa1841 +size 1412318725 diff --git a/pauhv4zk/episode_rosbags/cam_tf_world.npy b/pauhv4zk/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/pauhv4zk/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/pauhv4zk/episode_rosbags/depth_K.npy b/pauhv4zk/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/pauhv4zk/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/pauhv4zk/eval_robot.log b/pauhv4zk/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8fcdc2bfda9064343bb405128ab9f6c0a390225d --- /dev/null +++ b/pauhv4zk/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:05:44,607][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:05:44,610][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/u0i99mq0/.hydra/config.yaml b/u0i99mq0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/u0i99mq0/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/u0i99mq0/.hydra/hydra.yaml b/u0i99mq0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ef0495149c3691ba14cae6e0a75bb5118130136 --- /dev/null +++ b/u0i99mq0/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/u0i99mq0/.hydra/overrides.yaml b/u0i99mq0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/u0i99mq0/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/u0i99mq0/episode_rosbags/color_K.npy b/u0i99mq0/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/u0i99mq0/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/u0i99mq0/eval_video/0_eval.mp4 b/u0i99mq0/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a89900c000449b25d737bca5238de5dcc9220680 --- /dev/null +++ b/u0i99mq0/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64740432aabccb229691dc03f229cdce378022b23e11c906214e8cbd19130ab4 +size 1182738 diff --git a/u0i99mq0/eval_video/1_eval.mp4 b/u0i99mq0/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb8f5739cceb13cc0198bb00a4bfdc54d06550b8 --- /dev/null +++ b/u0i99mq0/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acff89a56e2528ff5ad34cdc12683c22969764353936fcc5f2f7be9124d3a956 +size 1199046 diff --git a/u0i99mq0/eval_video/2_eval.mp4 b/u0i99mq0/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5f19a7de76eac4c09d7b57b791209cb1897f039 --- /dev/null +++ b/u0i99mq0/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06d0e3b5afe6377da03819b20229825c528c59016fb72df3683840ec0745388d +size 1180627 diff --git a/u0i99mq0/eval_video/3_eval.mp4 b/u0i99mq0/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18d21af467ca5ff1a57c6fce754e484b718cef2d --- /dev/null +++ b/u0i99mq0/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee5c1dd7abeb860c96084d6e8d76c33f1020c26c9ec2e66ec3600b984d008a24 +size 1182765 diff --git a/u0i99mq0/eval_video/4_eval.mp4 b/u0i99mq0/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f44d9cebe78a174476097f167f66f9c36417ac91 --- /dev/null +++ b/u0i99mq0/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d212573bac6676e018bd385864578d09015dc286d13ed46d3c95270521397b55 +size 1227917 diff --git a/u0i99mq0/tb/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 b/u0i99mq0/tb/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 new file mode 100644 index 0000000000000000000000000000000000000000..ca316c86a61a753e36b0f865a639d8594a515f28 --- /dev/null +++ b/u0i99mq0/tb/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dc832106a3604e138b4a9fd704f43750f423e8ea7d513510a07c7ab3e3afd80 +size 6539 diff --git a/u0i99mq0/wandb/debug-internal.log b/u0i99mq0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c5fe1e6116840408423c339d80b8aa04afb8683a --- /dev/null +++ b/u0i99mq0/wandb/debug-internal.log @@ -0,0 +1,1277 @@ +2025-01-04 22:40:30,232 INFO StreamThr :2040553 [internal.py:wandb_internal():85] W&B internal server running at pid: 2040553, started at: 2025-01-04 22:40:30.231661 +2025-01-04 22:40:30,233 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:40:30,244 INFO WriterThread:2040553 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb +2025-01-04 22:40:30,245 DEBUG SenderThread:2040553 [sender.py:send():379] send: header +2025-01-04 22:40:30,254 DEBUG SenderThread:2040553 [sender.py:send():379] send: run +2025-01-04 22:40:30,497 INFO SenderThread:2040553 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files +2025-01-04 22:40:30,497 INFO SenderThread:2040553 [sender.py:_start_run_threads():1188] run started: u0i99mq0 with start time 1736048430.232252 +2025-01-04 22:40:30,511 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:40:30,511 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:40:30,587 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:40:30,592 DEBUG HandlerThread:2040553 [system_info.py:__init__():26] System info init +2025-01-04 22:40:30,592 DEBUG HandlerThread:2040553 [system_info.py:__init__():41] System info init done +2025-01-04 22:40:30,592 INFO HandlerThread:2040553 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:40:30,592 INFO SystemMonitor:2040553 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:40:30,592 INFO HandlerThread:2040553 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:40:30,593 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:40:30,593 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:40:30,593 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:40:30,594 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:40:30,595 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:40:30,604 DEBUG HandlerThread:2040553 [system_info.py:probe():152] Probing system +2025-01-04 22:40:30,606 DEBUG HandlerThread:2040553 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:40:30,610 DEBUG HandlerThread:2040553 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:40:30,610 DEBUG HandlerThread:2040553 [system_info.py:probe():200] Probing system done +2025-01-04 22:40:30,610 DEBUG HandlerThread:2040553 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:40:30.604775', 'startedAt': '2025-01-05T03:40:30.223272', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.418125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 781.2757759094238}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:40:30,610 INFO HandlerThread:2040553 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:40:30,610 INFO HandlerThread:2040553 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:40:30,611 DEBUG HandlerThread:2040553 [system_info.py:_save_code():44] Saving code +2025-01-04 22:40:30,612 DEBUG HandlerThread:2040553 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:40:30,612 DEBUG HandlerThread:2040553 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:40:30,674 DEBUG HandlerThread:2040553 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:40:30,674 INFO HandlerThread:2040553 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:40:30,678 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:40:30,678 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:40:30,678 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:40:30,678 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:40:30,920 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/yz3qi1zk-wandb-metadata.json +2025-01-04 22:40:30,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:40:30,936 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:40:30,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:40:30,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:40:30,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:30,938 INFO HandlerThread:2040553 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/tb" + save: true +} +control { + relay_id: "fefc616dc00e430c9f4883e9a3749183" +} +_info { + stream_id: "u0i99mq0" +} + +2025-01-04 22:40:30,939 DEBUG HandlerThread:2040553 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:40:30,942 INFO wandb-upload_2:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/t2a3afkd-diff.patch +2025-01-04 22:40:30,957 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/2fi1cuqv-code/FISH/eval_robot.py +2025-01-04 22:40:31,089 DEBUG SenderThread:2040553 [sender.py:send():379] send: telemetry +2025-01-04 22:40:31,089 DEBUG SenderThread:2040553 [sender.py:send():379] send: tbrecord +2025-01-04 22:40:31,089 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:40:31,089 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 with policy live +2025-01-04 22:40:31,268 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/uzviqvlo-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:40:31,498 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code +2025-01-04 22:40:31,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:32,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:33,498 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:33,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:34,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:35,417 DEBUG SenderThread:2040553 [sender.py:send():379] send: config +2025-01-04 22:40:35,417 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:35,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:36,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:37,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:37,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:38,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:39,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:40,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:40,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:41,500 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:41,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:42,949 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:43,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:44,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:45,501 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:45,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:40:45,936 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:40:45,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:46,062 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:46,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:47,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:48,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:49,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:50,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:51,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:51,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:52,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:53,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:54,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:55,504 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:55,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:56,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:56,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:57,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:58,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:59,505 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:59,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:00,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:00,936 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:00,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:01,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:02,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:02,506 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml +2025-01-04 22:41:02,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:03,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:04,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:05,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:06,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:07,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:07,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:08,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:09,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:10,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:11,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:12,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:12,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:13,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:14,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:15,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:15,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:15,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:16,942 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:17,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:18,067 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:18,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:19,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:20,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:21,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:22,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:23,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:23,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:24,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:25,181 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:41:25,182 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:41:25,182 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_64740432aabccb229691.mp4 with policy now +2025-01-04 22:41:25,182 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:41:25,182 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:41:25,183 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:41:25,422 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/ujjmgr91-media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:41:25,762 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/z5j8tha7-media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:41:25,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:26,944 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:27,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:27,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:28,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:29,705 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:29,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:30,595 DEBUG SystemMonitor:2040553 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:41:30,595 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:41:30,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:30,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:30,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:31,514 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:31,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:32,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:33,514 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:41:33,515 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:33,746 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/45uswq55-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:41:33,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:34,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:35,515 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:35,525 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:35,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:36,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:37,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:38,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:39,516 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:39,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:40,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:40,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:41,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:42,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:43,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:44,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:45,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:45,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:45,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:46,034 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:46,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:47,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:48,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:49,520 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:49,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:50,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:51,521 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:51,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:51,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:52,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:53,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:54,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:55,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:56,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:56,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:57,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:58,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:59,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:00,596 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:42:00,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:00,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:00,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:01,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:02,090 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:02,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:03,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:04,953 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:05,173 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:42:05,174 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:42:05,174 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 with policy now +2025-01-04 22:42:05,174 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:42:05,175 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:42:05,175 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:42:05,422 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/khjmzk3w-media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:42:05,525 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:42:05,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:05,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:42:05,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:42:05,778 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/4k1bv9tf-media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:42:05,966 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:06,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:07,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:07,942 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:07,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:08,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:09,527 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:09,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:10,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:11,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:12,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:13,205 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:13,528 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:13,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:14,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:15,529 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:42:15,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:15,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:15,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:16,345 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:42:16,345 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:42:16,345 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:42:16,529 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:42:16,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:17,530 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:17,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:18,346 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:18,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:19,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:20,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:21,531 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:21,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:22,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:23,944 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:23,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:24,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:25,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:26,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:27,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:28,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:29,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:29,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:30,596 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:42:30,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:30,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:30,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:31,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:32,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:33,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:34,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:35,058 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:35,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:36,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:37,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:38,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:39,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:40,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:40,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:41,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:42,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:43,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:44,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:45,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:45,938 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:45,985 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:46,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:46,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:47,474 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:42:47,474 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 with policy now +2025-01-04 22:42:47,475 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:42:47,475 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:42:47,475 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:42:47,476 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:42:47,540 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:42:47,540 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:47,540 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:42:47,541 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:42:47,725 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/y8g56jma-media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:42:47,969 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/twdltiak-media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:42:47,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:48,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:49,541 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:49,986 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:50,978 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:51,542 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:51,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:51,980 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:52,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:53,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:54,996 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:55,543 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:55,993 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:56,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:56,992 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:57,994 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:58,989 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:59,994 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:00,597 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:43:00,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:00,943 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:00,988 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:01,993 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:02,088 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:02,996 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:04,005 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:04,997 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:06,001 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:07,000 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:07,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:08,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:09,004 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:09,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:10,997 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:11,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:12,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:13,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:13,997 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:14,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:15,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:15,945 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:16,000 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:17,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:17,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:19,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:19,027 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:20,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:21,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:22,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:23,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:24,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:24,950 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:25,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:26,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:27,013 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:28,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:29,012 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:29,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:30,010 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:30,597 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:43:30,944 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:30,944 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:31,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:32,012 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:33,015 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:34,011 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:35,017 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:35,021 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:36,018 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:37,018 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:38,023 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:39,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:40,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:40,950 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:41,017 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:42,029 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:43,020 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:44,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:45,019 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:45,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:45,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:45,952 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:46,020 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:47,023 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:48,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:49,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:50,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:50,954 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:51,027 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:52,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:53,031 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:54,028 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:55,031 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:56,032 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:56,954 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:57,038 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:58,034 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:59,032 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:00,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:00,598 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:44:00,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:00,945 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:01,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:02,040 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:02,090 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:03,041 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:04,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:05,046 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:06,044 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:07,044 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:07,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:08,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:09,047 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:10,046 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:11,050 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:12,051 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:12,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:13,054 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:14,057 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:15,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:15,949 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:15,949 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:16,055 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:17,057 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:18,059 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:18,084 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:19,064 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:20,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:21,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:22,055 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:23,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:23,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:24,064 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:25,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:26,068 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:27,065 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:28,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:28,965 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:29,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:30,059 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:30,598 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:44:30,955 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:30,955 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:31,062 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:32,066 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:33,065 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:34,050 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:34,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:35,063 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:36,062 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:37,078 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:38,075 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:39,080 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:39,968 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:40,078 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:41,079 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:42,081 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:43,079 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:44,081 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:44,972 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:45,079 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:45,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:45,948 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:46,083 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:47,084 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:48,083 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:49,084 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:50,087 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:50,099 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:51,089 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:52,091 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:53,092 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:54,089 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:55,092 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:55,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:56,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:57,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:58,093 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:59,097 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:00,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:00,599 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:45:00,949 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:00,949 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:01,036 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:01,099 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:02,098 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:03,099 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:04,102 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:05,100 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:06,103 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:06,967 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:07,103 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:08,112 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:09,112 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:10,111 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:11,117 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:12,116 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:12,967 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:13,121 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:14,119 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:15,120 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:15,953 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:15,953 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:16,128 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:17,121 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:18,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:18,122 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:19,123 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:20,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:21,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:22,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:23,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:23,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:24,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:25,131 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:26,124 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:27,130 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:28,129 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:29,131 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:29,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:30,126 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:30,599 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:45:30,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:30,952 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:31,129 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:32,127 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:33,133 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:34,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:35,044 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:35,132 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:36,134 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:37,134 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:38,136 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:39,136 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:40,137 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:40,972 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:41,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:42,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:43,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:44,138 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:45,144 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:45,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:45,958 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:46,069 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:46,142 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:47,138 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:48,146 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:49,148 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:50,148 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:51,151 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:51,974 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:52,145 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:53,151 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:54,152 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:55,150 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:56,151 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:56,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:57,154 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:58,152 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:59,152 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:00,158 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:00,599 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:46:00,955 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:00,956 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:01,154 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:02,088 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:02,154 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:03,157 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:04,158 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:05,155 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:06,158 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:07,162 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:07,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:08,157 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:09,160 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:10,160 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:11,162 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:12,165 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:12,983 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:13,166 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:14,168 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:15,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:15,964 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:15,964 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:16,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:17,172 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:18,052 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:18,167 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:19,167 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:20,166 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:21,168 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:22,169 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:23,176 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:23,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:24,172 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:25,170 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:26,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:27,174 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:28,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:29,173 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:29,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:30,178 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:30,600 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:46:30,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:30,960 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:31,177 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:32,181 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:33,177 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:34,178 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:35,078 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:35,184 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:36,186 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:37,185 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:38,186 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:39,190 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:40,191 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:40,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:41,187 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:42,191 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:43,193 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:44,189 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:45,191 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:45,955 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:45,955 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:46,107 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:46,188 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:47,189 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:48,192 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:49,193 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:50,197 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:51,206 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:51,989 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:52,196 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:53,203 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:54,202 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:55,204 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:56,199 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:57,202 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:57,988 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:58,203 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:59,207 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:00,207 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:00,600 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:47:00,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:00,958 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:01,202 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:02,208 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:03,063 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:03,208 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:04,208 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:05,206 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:06,205 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:07,209 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:08,210 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:08,995 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:09,215 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:10,210 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:11,210 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:12,211 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:13,220 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:13,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:14,219 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:15,215 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:15,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:15,957 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:16,218 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:17,219 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:18,226 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:19,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:19,222 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:20,223 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:21,221 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:22,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:23,227 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:24,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:24,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:25,225 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:26,226 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:27,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:28,227 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:29,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:29,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:30,225 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:30,601 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:47:30,965 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:30,965 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:31,225 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:32,233 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:33,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:34,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:35,111 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:35,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:36,233 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:37,234 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:38,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:39,236 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:40,236 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:40,996 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:41,234 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:42,241 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:43,237 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:44,238 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:45,239 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:45,964 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:45,964 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:46,038 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:46,244 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:47,239 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:48,240 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:49,240 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:50,246 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:51,241 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:52,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:52,248 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:53,245 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:54,250 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:55,255 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:56,246 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:57,249 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:57,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:58,247 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:59,247 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:00,249 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:00,601 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:48:00,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:00,961 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:01,248 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:02,249 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:03,066 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:03,253 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:04,255 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:05,251 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:06,250 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:07,254 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:08,251 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:09,004 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:09,254 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:10,255 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:11,253 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:12,253 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:13,262 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:14,011 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:14,257 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:15,256 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:15,966 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:15,966 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:16,261 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:17,260 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:18,261 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:19,116 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:19,264 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:20,271 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:21,265 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:22,267 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:23,272 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:24,270 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:25,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:25,268 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:26,276 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:27,271 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:28,268 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:29,272 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:30,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:30,271 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:30,602 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:48:30,978 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:30,978 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:31,273 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:32,272 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:33,273 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:34,275 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:35,108 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:35,275 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:36,284 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:37,286 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:38,290 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:39,286 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:40,291 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:41,013 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:41,290 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:42,292 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:43,294 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:44,298 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:45,295 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:45,972 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:45,972 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:46,119 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:46,299 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:47,304 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:48,297 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:49,297 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:50,299 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:51,311 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:52,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:52,306 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:53,300 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:54,304 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:55,306 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:56,305 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:57,013 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:57,302 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:58,303 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:59,310 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:00,302 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:00,602 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:49:00,968 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:00,969 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:01,305 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:02,082 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:02,305 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:03,307 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:04,307 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:05,306 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:06,307 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:07,308 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:08,021 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:08,309 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:09,308 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:10,312 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:11,315 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:12,316 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:13,315 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:14,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:14,311 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:15,310 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:15,973 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:15,973 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:16,316 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:17,316 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:18,315 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:19,070 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:19,318 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:20,319 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:21,317 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:22,320 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:23,318 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:24,320 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:25,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:25,322 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:26,322 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:27,322 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:28,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:29,323 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:29,635 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:49:29,635 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:49:29,636 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:49:29,679 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:49:29,679 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:49:29,932 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/iuh8sx1j-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:49:30,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:30,401 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:30,603 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:49:30,970 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:30,970 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:31,329 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:31,679 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:32,323 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:33,323 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:33,680 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:34,326 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:35,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:35,681 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:35,826 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:36,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:37,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:38,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:39,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:40,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:41,010 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:41,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:42,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:43,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:44,327 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:45,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:45,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:45,969 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:46,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:46,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:47,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:48,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:49,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:49,686 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:50,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:51,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:51,686 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:52,010 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:52,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:53,330 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:54,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:55,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:56,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:57,011 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:57,332 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:58,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:59,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:00,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:00,603 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:50:00,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:00,970 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:01,332 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:02,321 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:02,333 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:02,873 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:02,873 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:02,873 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 with policy now +2025-01-04 22:50:02,874 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:02,874 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:02,874 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:03,093 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/qyjas31q-media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:50:03,333 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:03,691 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:03,691 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:03,692 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:50:03,692 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:50:03,973 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/7s0lyx8z-media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:50:04,336 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:04,692 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:05,337 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:06,348 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:07,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:07,693 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:08,161 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:08,344 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:09,340 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:09,693 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:10,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:11,353 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:12,346 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:13,348 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:14,020 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:14,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:14,690 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:14,690 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:14,690 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:14,695 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:14,695 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:15,346 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:15,696 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:15,976 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:15,976 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:16,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:17,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:17,696 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:18,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:19,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:19,693 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:19,697 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:20,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:21,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:22,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:23,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:24,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:25,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:25,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:26,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:27,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:27,700 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:28,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:29,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:29,700 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:30,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:30,603 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:50:30,605 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:30,970 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:30,970 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:31,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:32,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:33,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:34,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:35,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:35,702 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:36,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:36,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:37,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:37,703 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:38,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:39,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:39,704 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:40,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:41,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:41,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:41,705 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:42,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:43,353 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:43,706 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:44,353 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:45,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:45,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:45,980 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:46,027 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:46,360 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:46,725 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:46,725 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 with policy now +2025-01-04 22:50:46,727 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:46,728 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:46,728 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:46,728 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:46,948 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/qdd3b483-media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:50:47,362 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:47,707 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:47,707 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:47,707 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:50:47,708 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:50:47,988 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/0imva0y3-media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:50:48,380 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:49,360 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:50,360 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:51,367 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:51,708 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:52,023 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:52,358 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:53,365 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:53,709 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:54,369 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:55,378 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:56,375 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:57,025 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:57,372 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:57,481 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:50:57,482 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:50:57,687 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table +2025-01-04 22:50:57,907 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/wr0kp33z-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:50:57,934 INFO wandb-upload_2:2040553 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6fqhwqg_ +2025-01-04 22:50:57,986 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/g4jl9fke-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:58,133 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:50:58,142 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:58,327 INFO SenderThread:2040553 [sender.py:send_request_log_artifact():1525] logged artifact run-u0i99mq0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIwNjY3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NTMzMA==', 'latestArtifact': None}} +2025-01-04 22:50:58,328 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:58,328 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-04 22:50:58,328 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:58,328 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,329 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,329 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:58,329 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,329 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,330 DEBUG SenderThread:2040553 [sender.py:send():379] send: telemetry +2025-01-04 22:50:58,330 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,331 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,331 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,332 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,332 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,332 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,333 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,333 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,333 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,333 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,334 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,334 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,334 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,334 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,335 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,335 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,335 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,335 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,335 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,336 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,336 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,336 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,336 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:50:58,365 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:58,550 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/z2rvkuk3-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:50:58,677 INFO wandb-upload_1:2040553 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwcd1hnub +2025-01-04 22:50:58,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:58,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:58,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:50:58,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval +2025-01-04 22:50:59,063 INFO SenderThread:2040553 [sender.py:send_request_log_artifact():1525] logged artifact run-u0i99mq0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIwNjY3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NTMzMQ==', 'latestArtifact': None}} +2025-01-04 22:50:59,063 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:59,063 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json with policy now +2025-01-04 22:50:59,064 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:59,064 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:59,064 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:59,247 INFO wandb-upload_2:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/cjn02b2n-media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:50:59,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:59,476 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/6jeypiub-media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:50:59,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:59,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:00,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:00,604 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:51:00,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:51:00,972 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:51:01,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:02,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:02,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:03,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:04,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:05,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:06,367 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:07,367 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:08,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:08,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:09,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:10,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:11,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:12,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:12,697 DEBUG SenderThread:2040553 [sender.py:send():379] send: exit +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_exit():588] handling runtime: 642 +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_exit():594] send defer +2025-01-04 22:51:12,698 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,698 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:51:12,698 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:51:12,699 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,699 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:51:12,699 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,699 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:51:12,699 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:51:12,699 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,699 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:51:12,699 INFO HandlerThread:2040553 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:51:12,699 DEBUG SystemMonitor:2040553 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:51:12,699 DEBUG SystemMonitor:2040553 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:51:12,700 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:51:12,700 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:51:12,717 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:51:12,785 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:51:12,785 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:51:12,785 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:51:12,786 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,786 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:51:12,786 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:51:12,786 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:51:12,786 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,786 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:51:12,786 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,786 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:51:12,787 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:51:12,787 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,787 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:51:13,718 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:19,748 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,748 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:51:19,748 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:51:19,748 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:19,748 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:51:19,749 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:51:19,749 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:19,749 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:51:19,750 DEBUG SenderThread:2040553 [sender.py:send():379] send: summary +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:51:19,750 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:51:19,751 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:19,751 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:51:19,751 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,751 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:51:19,754 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:19,855 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:51:19,856 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:19,856 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:51:19,856 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,856 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:51:20,699 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:20,720 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:51:20,721 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml +2025-01-04 22:51:21,862 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:51:21,862 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:21,863 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:21,863 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:51:21,863 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:21,863 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:51:21,863 INFO SenderThread:2040553 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:51:21,865 INFO SenderThread:2040553 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:51:21,867 INFO SenderThread:2040553 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:51:21,882 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:51:21,882 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:21,882 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:51:21,882 DEBUG SenderThread:2040553 [sender.py:send():379] send: artifact +2025-01-04 22:51:22,062 INFO SenderThread:2040553 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:51:22,062 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:22,062 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:51:22,062 INFO SenderThread:2040553 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:51:22,700 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml config.yaml +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json wandb-summary.json +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt requirements.txt +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log output.log +2025-01-04 22:51:22,723 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch diff.patch +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:51:22,729 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:51:22,729 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:22,731 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:22,732 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:51:22,733 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:22,733 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:51:22,733 INFO SenderThread:2040553 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:51:22,904 INFO wandb-upload_2:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml +2025-01-04 22:51:22,906 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:51:22,949 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/osc069yn-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:51:22,957 INFO wandb-upload_3:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt +2025-01-04 22:51:22,985 INFO wandb-upload_4:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:23,186 INFO Thread-11 (_thread_body):2040553 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:51:23,186 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,186 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:51:23,186 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,186 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:51:23,187 INFO SenderThread:2040553 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:51:23,188 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:51:23,188 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,188 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:51:23,188 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,188 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:51:23,188 INFO SenderThread:2040553 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:51:23,479 INFO SenderThread:2040553 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:51:23,479 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:51:23,479 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,479 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,480 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:51:23,480 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:51:23,480 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,480 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send():379] send: final +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send():379] send: footer +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,480 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:51:23,481 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:23,481 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:23,481 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:51:23,482 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:23,482 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:51:23,483 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:51:23,484 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:51:23,484 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:23,565 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:23,566 INFO MainThread:2040553 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:51:23,566 INFO MainThread:2040553 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:51:23,566 INFO MainThread:2040553 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:51:23,567 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:51:23,567 INFO HandlerThread:2040553 [handler.py:finish():882] shutting down handler +2025-01-04 22:51:24,482 INFO WriterThread:2040553 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb +2025-01-04 22:51:24,566 INFO SenderThread:2040553 [sender.py:finish():1615] shutting down sender +2025-01-04 22:51:24,566 INFO SenderThread:2040553 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:51:24,566 INFO SenderThread:2040553 [file_pusher.py:join():175] waiting for file pusher diff --git a/u0i99mq0/wandb/debug.log b/u0i99mq0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..76224a41211001deb425e06406307614344fb772 --- /dev/null +++ b/u0i99mq0/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Configure stats pid to 2040388 +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/settings +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug.log +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug-internal.log +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0'} +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():619] starting backend +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():623] setting up manager +2025-01-04 22:40:30,230 INFO MainThread:2040388 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:40:30,231 INFO MainThread:2040388 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:40:30,244 INFO MainThread:2040388 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:40:30,251 INFO MainThread:2040388 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:40:30,510 INFO MainThread:2040388 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:40:30,581 INFO MainThread:2040388 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:40:30,581 INFO MainThread:2040388 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:40:30,936 INFO MainThread:2040388 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:40:30,936 INFO MainThread:2040388 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:40:30,937 INFO MainThread:2040388 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:40:30,937 INFO MainThread:2040388 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:40:30,938 INFO MainThread:2040388 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:40:30,938 INFO MainThread:2040388 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/tb, True +2025-01-04 22:40:35,416 INFO MainThread:2040388 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:51:24,567 WARNING MsgRouterThr:2040388 [router.py:message_loop():77] message_loop has been closed diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b351f82c3537b230bb480f52d33908c37f6fac46 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736048430 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..42eb89b1c237af51a986abf7c0492187b965280c --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[52, 60, 0.8666666666666667, 0.7379104941607537, 0.9464898704589503, 0.7666581049028185, 0.9401788226379755, 0.7841424270896062, 0.9246097057208565, 0.7958142049018986, 0.9194757360126062]]} \ No newline at end of file diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a89900c000449b25d737bca5238de5dcc9220680 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64740432aabccb229691dc03f229cdce378022b23e11c906214e8cbd19130ab4 +size 1182738 diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb8f5739cceb13cc0198bb00a4bfdc54d06550b8 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acff89a56e2528ff5ad34cdc12683c22969764353936fcc5f2f7be9124d3a956 +size 1199046 diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5f19a7de76eac4c09d7b57b791209cb1897f039 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06d0e3b5afe6377da03819b20229825c528c59016fb72df3683840ec0745388d +size 1180627 diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18d21af467ca5ff1a57c6fce754e484b718cef2d --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee5c1dd7abeb860c96084d6e8d76c33f1020c26c9ec2e66ec3600b984d008a24 +size 1182765 diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f44d9cebe78a174476097f167f66f9c36417ac91 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d212573bac6676e018bd385864578d09015dc286d13ed46d3c95270521397b55 +size 1227917 diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dc6082f36adb85d606e793e0d9c04a772c609323 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log @@ -0,0 +1,277 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736048435.417923]: resetting environment +[INFO] [1736048435.423231]: cleared current plan +[INFO] [1736048435.424123]: moving to home +[INFO] [1736048436.425209]: reached home +[INFO] [1736048436.427351]: reset action history +[INFO] [1736048439.134948]: environment reset +Starting episode 0 +[INFO] [1736048439.136795]: resetting environment +[INFO] [1736048439.141048]: cleared current plan +[INFO] [1736048439.142107]: moving to home +[INFO] [1736048440.143653]: reached home +[INFO] [1736048440.146212]: reset action history +[INFO] [1736048442.905783]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736048453.464511]: resetting environment +[INFO] [1736048453.469655]: cleared current plan +[INFO] [1736048453.471098]: moving to home +[INFO] [1736048454.473268]: reached home +[INFO] [1736048454.474975]: reset action history +[INFO] [1736048457.182180]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/episode_rosbags/episode_0_2025-01-04-22-40-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2040388.log +started roslaunch server http://158.130.50.37:39613/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2040676] +process[print_text-2]: started with pid [2040677] +process[rosbag_record-3]: started with pid [2040678] +started bagging! +[INFO] [1736048484.222429]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736048484.940647]: Stored episode 1. +[INFO] [1736048484.941992]: Saving video... +[INFO] [1736048485.137046]: Video saved! +Starting episode 1 +[INFO] [1736048485.149505]: resetting environment +[INFO] [1736048485.159846]: cleared current plan +[INFO] [1736048485.160444]: moving to home +[INFO] [1736048488.663523]: reached home +[INFO] [1736048488.664047]: reset action history +[INFO] [1736048491.371312]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048493.470410]: resetting environment +[INFO] [1736048493.476655]: cleared current plan +[INFO] [1736048493.476981]: moving to home +[INFO] [1736048494.478507]: reached home +[INFO] [1736048494.478914]: reset action history +[INFO] [1736048497.185485]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/episode_rosbags/episode_1_2025-01-04-22-41-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2040388.log +started roslaunch server http://158.130.50.37:33805/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2040786] +process[print_text-5]: started with pid [2040787] +process[rosbag_record-6]: started with pid [2040811] +started bagging! +For topic gripper_width: timestamp difference is -118322546 for nearest: 1736048508151552265 - target: 1736048508269874811 at idx 142 +For topic gripper_width: timestamp difference is -116784091 for nearest: 1736048510151507144 - target: 1736048510268291235 at idx 166 +[INFO] [1736048524.198050]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736048524.913092]: Stored episode 2. +[INFO] [1736048524.913598]: Saving video... +[INFO] [1736048525.127305]: Video saved! +Starting episode 2 +[INFO] [1736048525.152260]: resetting environment +[INFO] [1736048525.166076]: cleared current plan +[INFO] [1736048525.168046]: moving to home +[INFO] [1736048528.377367]: reached home +[INFO] [1736048528.379475]: reset action history +[INFO] [1736048531.156812]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048535.253292]: resetting environment +[INFO] [1736048535.261502]: cleared current plan +[INFO] [1736048535.262006]: moving to home +[INFO] [1736048536.267061]: reached home +[INFO] [1736048536.267524]: reset action history +[INFO] [1736048538.975033]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/episode_rosbags/episode_2_2025-01-04-22-42-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2040388.log +started roslaunch server http://158.130.50.37:43239/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2040923] +process[print_text-8]: started with pid [2040924] +process[rosbag_record-9]: started with pid [2040948] +started bagging! +[INFO] [1736048565.990499]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736048566.710704]: Stored episode 3. +[INFO] [1736048566.712240]: Saving video... +[INFO] [1736048567.431255]: Video saved! +Starting episode 3 +[INFO] [1736048567.477192]: resetting environment +[INFO] [1736048567.484460]: cleared current plan +[INFO] [1736048567.484655]: moving to home +[WARN] [1736048567.365501]: Plan exhausted +[WARN] [1736048567.409314]: Plan exhausted +[WARN] [1736048567.443786]: Plan exhausted +[WARN] [1736048567.483335]: Plan exhausted +[INFO] [1736048570.687936]: reached home +[INFO] [1736048570.694453]: reset action history +[INFO] [1736048573.473424]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736048969.376040]: resetting environment +[INFO] [1736048969.392482]: cleared current plan +[INFO] [1736048969.393849]: moving to home +[INFO] [1736048970.399214]: reached home +[INFO] [1736048970.408596]: reset action history +[INFO] [1736048973.125385]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/episode_rosbags/episode_3_2025-01-04-22-49-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2040388.log +started roslaunch server http://158.130.50.37:42323/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2041198] +process[print_text-11]: started with pid [2041199] +process[rosbag_record-12]: started with pid [2041200] +started bagging! +For topic gripper_width: timestamp difference is 100910645 for nearest: 1736048987951458674 - target: 1736048987850548029 at idx 201 +For topic gripper_width: timestamp difference is 102447815 for nearest: 1736048989551495857 - target: 1736048989449048042 at idx 220 +[INFO] [1736049001.536288]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736049002.321005]: Stored episode 4. +[INFO] [1736049002.324029]: Saving video... +[INFO] [1736049002.829061]: Video saved! +Starting episode 4 +[INFO] [1736049002.874275]: resetting environment +[INFO] [1736049002.894273]: cleared current plan +[INFO] [1736049002.896179]: moving to home +[WARN] [1736049002.892387]: Plan exhausted +[INFO] [1736049006.101436]: reached home +[INFO] [1736049006.117377]: reset action history +[INFO] [1736049008.890620]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049014.616462]: resetting environment +[INFO] [1736049014.632991]: cleared current plan +[INFO] [1736049014.636779]: moving to home +[INFO] [1736049015.642226]: reached home +[INFO] [1736049015.645269]: reset action history +[INFO] [1736049018.419227]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/episode_rosbags/episode_4_2025-01-04-22-50-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2040388.log +started roslaunch server http://158.130.50.37:35725/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2041312] +process[print_text-14]: started with pid [2041313] +process[rosbag_record-15]: started with pid [2041338] +started bagging! +For topic gripper_width: timestamp difference is 101128540 for nearest: 1736049026018167219 - target: 1736049025917038679 at idx 114 +For topic gripper_width: timestamp difference is 104313223 for nearest: 1736049027618119566 - target: 1736049027513806343 at idx 136 +For topic gripper_width: timestamp difference is 100413851 for nearest: 1736049034218206934 - target: 1736049034117793083 at idx 220 +For topic gripper_width: timestamp difference is 105317834 for nearest: 1736049036618162873 - target: 1736049036512845039 at idx 249 +For topic gripper_width: timestamp difference is 109442155 for nearest: 1736049038618186156 - target: 1736049038508744001 at idx 249 +For topic gripper_width: timestamp difference is 112688336 for nearest: 1736049040218161616 - target: 1736049040105473280 at idx 249 +For topic gripper_width: timestamp difference is 114357876 for nearest: 1736049041018163847 - target: 1736049040903805971 at idx 249 +For topic gripper_width: timestamp difference is 148279883 for nearest: 1736049041351495482 - target: 1736049041203215599 at idx 249 +[INFO] [1736049045.415117]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736049046.174076]: Stored episode 5. +[INFO] [1736049046.182769]: Saving video... +[INFO] [1736049046.678308]: Video saved! +[INFO] [1736049046.728483]: resetting environment +[INFO] [1736049046.746871]: cleared current plan +[INFO] [1736049046.747647]: moving to home +[INFO] [1736049049.858209]: reached home +[INFO] [1736049049.866122]: reset action history +[INFO] [1736049052.674025]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2058b681eea3a9727e05d2ae92d43452f642bf1e --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:40:30.604775", + "startedAt": "2025-01-05T03:40:30.223272", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.418125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.005, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 781.2757759094238 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a15e3822657c1cf08777202c68c26e53c5d0f5 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "64740432aabccb229691dc03f229cdce378022b23e11c906214e8cbd19130ab4", "size": 1182738, "path": "media/videos/eval/0_eval_0_64740432aabccb229691.mp4"}, "global_step": 665, "_timestamp": 1736049058.0404217, "_runtime": 627.8081698417664, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "acff89a56e2528ff5ad34cdc12683c22969764353936fcc5f2f7be9124d3a956", "size": 1199046, "path": "media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4"}, "eval/num_success": 1.0, "episode": 60.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "06d0e3b5afe6377da03819b20229825c528c59016fb72df3683840ec0745388d", "size": 1180627, "path": "media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ee5c1dd7abeb860c96084d6e8d76c33f1020c26c9ec2e66ec3600b984d008a24", "size": 1182765, "path": "media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d212573bac6676e018bd385864578d09015dc286d13ed46d3c95270521397b55", "size": 1227917, "path": "media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://mhy8rlby2906qk49zi2x9zjgqe2d9rkp3miifve4mlcmcn7nb7frzsex88fg5qsaxtpa38otcs3lpm4uq5pi6jpi3tw9e9flae20z4iikrgh10i943fdv6x3l656s40w/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ipgcukgtylvgk2rlbn0ketm3qanc40btyc8hk2nmxq2g0zdwd5y1prejutl9z88n0uacbn1zmes98oezjpytnfjyj1x0zcouowiwts8usavqbfex61psu5t5d2ukicm2:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 52, "total_num_episodes": 60, "total_success_rate": 0.8666666666666667, "total_success_rate_lb_0.01": 0.7379104941607537, "total_success_rate_ub_0.01": 0.9464898704589503, "total_success_rate_lb_0.025": 0.7666581049028185, "total_success_rate_ub_0.025": 0.9401788226379755, "total_success_rate_lb_0.05": 0.7841424270896062, "total_success_rate_ub_0.05": 0.9246097057208565, "total_success_rate_lb_0.1": 0.7958142049018986, "total_success_rate_ub_0.1": 0.9194757360126062, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "22e3aa8cab918fc471a4e0a345e20611146616064e2451ab287185ee48d1f92c", "size": 520, "artifact_path": "wandb-client-artifact://wqffzmbmjxjsy3razhsinudvdn6ge83gav8p4ewzx4ytquwc03zu0a8k115rccfwh3rnj84ij5pzyskqr6cq545vmhg9vw3apupc6hv6dj30kcgzg7ywxp25vxj5346v/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xyglz6oz5a49nyuq3hvktwurd6qh5w832rcch369bakh5lfq6tm97fpdswwiksfzq4cfsf0pupq9kflgdbjdhw7syw1qryulspqfp3lb3fscm0x47t2reu6bvrkag90u:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 642}, "eval/total_success_rate": 0.8666666746139526, "eval/total_success_rate_lb_0.01": 0.7379105091094971, "eval/total_success_rate_ub_0.01": 0.9464898705482483, "eval/total_success_rate_lb_0.025": 0.7666581273078918, "eval/total_success_rate_ub_0.025": 0.9401788115501404, "eval/total_success_rate_lb_0.05": 0.7841424345970154, "eval/total_success_rate_ub_0.05": 0.9246097207069397, "eval/total_success_rate_lb_0.1": 0.7958142161369324, "eval/total_success_rate_ub_0.1": 0.9194757342338562} \ No newline at end of file diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug-internal.log b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c5fe1e6116840408423c339d80b8aa04afb8683a --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug-internal.log @@ -0,0 +1,1277 @@ +2025-01-04 22:40:30,232 INFO StreamThr :2040553 [internal.py:wandb_internal():85] W&B internal server running at pid: 2040553, started at: 2025-01-04 22:40:30.231661 +2025-01-04 22:40:30,233 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:40:30,244 INFO WriterThread:2040553 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb +2025-01-04 22:40:30,245 DEBUG SenderThread:2040553 [sender.py:send():379] send: header +2025-01-04 22:40:30,254 DEBUG SenderThread:2040553 [sender.py:send():379] send: run +2025-01-04 22:40:30,497 INFO SenderThread:2040553 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files +2025-01-04 22:40:30,497 INFO SenderThread:2040553 [sender.py:_start_run_threads():1188] run started: u0i99mq0 with start time 1736048430.232252 +2025-01-04 22:40:30,511 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:40:30,511 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:40:30,587 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:40:30,592 DEBUG HandlerThread:2040553 [system_info.py:__init__():26] System info init +2025-01-04 22:40:30,592 DEBUG HandlerThread:2040553 [system_info.py:__init__():41] System info init done +2025-01-04 22:40:30,592 INFO HandlerThread:2040553 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:40:30,592 INFO SystemMonitor:2040553 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:40:30,592 INFO HandlerThread:2040553 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:40:30,593 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:40:30,593 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:40:30,593 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:40:30,594 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:40:30,595 INFO SystemMonitor:2040553 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:40:30,604 DEBUG HandlerThread:2040553 [system_info.py:probe():152] Probing system +2025-01-04 22:40:30,606 DEBUG HandlerThread:2040553 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:40:30,610 DEBUG HandlerThread:2040553 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:40:30,610 DEBUG HandlerThread:2040553 [system_info.py:probe():200] Probing system done +2025-01-04 22:40:30,610 DEBUG HandlerThread:2040553 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:40:30.604775', 'startedAt': '2025-01-05T03:40:30.223272', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.418125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 781.2757759094238}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:40:30,610 INFO HandlerThread:2040553 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:40:30,610 INFO HandlerThread:2040553 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:40:30,611 DEBUG HandlerThread:2040553 [system_info.py:_save_code():44] Saving code +2025-01-04 22:40:30,612 DEBUG HandlerThread:2040553 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:40:30,612 DEBUG HandlerThread:2040553 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:40:30,674 DEBUG HandlerThread:2040553 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:40:30,674 INFO HandlerThread:2040553 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:40:30,678 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:40:30,678 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:40:30,678 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:40:30,678 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:40:30,920 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/yz3qi1zk-wandb-metadata.json +2025-01-04 22:40:30,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:40:30,936 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:40:30,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:40:30,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:40:30,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:30,938 INFO HandlerThread:2040553 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/tb" + save: true +} +control { + relay_id: "fefc616dc00e430c9f4883e9a3749183" +} +_info { + stream_id: "u0i99mq0" +} + +2025-01-04 22:40:30,939 DEBUG HandlerThread:2040553 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:40:30,942 INFO wandb-upload_2:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/t2a3afkd-diff.patch +2025-01-04 22:40:30,957 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/2fi1cuqv-code/FISH/eval_robot.py +2025-01-04 22:40:31,089 DEBUG SenderThread:2040553 [sender.py:send():379] send: telemetry +2025-01-04 22:40:31,089 DEBUG SenderThread:2040553 [sender.py:send():379] send: tbrecord +2025-01-04 22:40:31,089 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:40:31,089 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 with policy live +2025-01-04 22:40:31,268 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/uzviqvlo-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:40:31,498 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH +2025-01-04 22:40:31,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code +2025-01-04 22:40:31,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:32,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:33,498 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:33,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:34,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:35,417 DEBUG SenderThread:2040553 [sender.py:send():379] send: config +2025-01-04 22:40:35,417 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:35,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:36,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:37,499 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:37,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:38,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:39,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:40,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:40,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:41,500 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:41,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:42,949 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:43,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:44,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:45,501 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:45,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:40:45,936 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:40:45,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:46,062 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:46,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:47,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:48,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:49,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:50,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:51,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:51,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:52,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:53,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:54,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:55,504 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:55,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:56,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:40:56,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:57,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:58,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:40:59,505 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:40:59,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:00,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:00,936 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:00,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:01,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:02,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:02,506 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml +2025-01-04 22:41:02,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:03,940 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:04,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:05,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:06,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:07,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:07,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:08,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:09,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:10,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:11,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:12,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:12,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:13,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:14,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:15,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:15,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:15,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:16,942 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:17,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:18,067 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:18,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:19,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:20,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:21,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:22,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:23,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:23,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:24,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:25,181 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:41:25,182 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:41:25,182 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_64740432aabccb229691.mp4 with policy now +2025-01-04 22:41:25,182 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:41:25,182 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:41:25,183 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:41:25,422 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/ujjmgr91-media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos +2025-01-04 22:41:25,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:41:25,762 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/z5j8tha7-media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:41:25,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:26,944 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:27,512 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:27,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:28,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:29,705 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:29,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:30,595 DEBUG SystemMonitor:2040553 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:41:30,595 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:41:30,936 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:30,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:30,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:31,514 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:31,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:32,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:33,514 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:41:33,515 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:33,746 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/45uswq55-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:41:33,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:34,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:35,515 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:35,525 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:35,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:36,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:37,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:38,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:39,516 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:39,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:40,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:40,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:41,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:42,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:43,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:44,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:45,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:41:45,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:41:45,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:46,034 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:46,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:47,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:48,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:49,520 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:49,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:50,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:51,521 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:41:51,938 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:51,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:52,947 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:53,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:54,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:55,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:56,939 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:41:56,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:57,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:58,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:41:59,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:00,596 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:42:00,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:00,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:00,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:01,951 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:02,090 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:02,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:03,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:04,953 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:05,173 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:42:05,174 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:42:05,174 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 with policy now +2025-01-04 22:42:05,174 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:42:05,175 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:42:05,175 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:42:05,422 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/khjmzk3w-media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:42:05,525 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:42:05,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:05,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:42:05,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:42:05,778 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/4k1bv9tf-media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:42:05,966 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:06,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:07,526 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:07,942 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:07,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:08,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:09,527 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:09,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:10,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:11,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:12,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:13,205 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:13,528 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:13,956 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:14,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:15,529 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:42:15,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:15,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:15,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:16,345 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:42:16,345 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:42:16,345 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:42:16,529 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:42:16,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:17,530 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:17,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:18,346 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:18,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:19,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:20,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:21,531 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:21,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:22,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:23,944 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:23,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:24,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:25,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:26,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:27,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:28,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:29,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:29,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:30,596 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:42:30,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:30,937 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:30,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:31,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:32,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:33,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:34,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:35,058 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:35,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:36,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:37,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:38,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:39,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:40,941 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:40,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:41,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:42,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:43,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:44,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:45,937 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:42:45,938 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:42:45,985 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:46,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:46,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:47,474 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:42:47,474 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 with policy now +2025-01-04 22:42:47,475 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:42:47,475 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:42:47,475 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:42:47,476 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:42:47,540 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:42:47,540 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:47,540 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:42:47,541 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:42:47,725 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/y8g56jma-media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:42:47,969 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/twdltiak-media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:42:47,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:48,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:49,541 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:49,986 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:50,978 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:51,542 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:51,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:51,980 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:52,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:53,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:54,996 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:55,543 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:42:55,993 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:56,946 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:42:56,992 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:57,994 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:58,989 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:42:59,994 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:00,597 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:43:00,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:00,943 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:00,988 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:01,993 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:02,088 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:02,996 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:04,005 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:04,997 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:06,001 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:07,000 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:07,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:08,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:09,004 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:09,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:10,997 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:11,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:12,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:13,943 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:13,997 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:14,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:15,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:15,945 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:16,000 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:17,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:17,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:19,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:19,027 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:20,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:21,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:22,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:23,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:24,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:24,950 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:25,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:26,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:27,013 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:28,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:29,012 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:29,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:30,010 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:30,597 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:43:30,944 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:30,944 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:31,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:32,012 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:33,015 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:34,011 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:35,017 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:35,021 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:36,018 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:37,018 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:38,023 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:39,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:40,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:40,950 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:41,017 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:42,029 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:43,020 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:44,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:45,019 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:45,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:45,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:43:45,952 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:43:46,020 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:47,023 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:48,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:49,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:50,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:50,954 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:51,027 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:52,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:53,031 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:54,028 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:55,031 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:56,032 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:56,954 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:43:57,038 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:58,034 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:43:59,032 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:00,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:00,598 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:44:00,945 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:00,945 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:01,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:02,040 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:02,090 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:03,041 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:04,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:05,046 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:06,044 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:07,044 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:07,960 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:08,042 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:09,047 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:10,046 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:11,050 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:12,051 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:12,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:13,054 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:14,057 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:15,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:15,949 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:15,949 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:16,055 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:17,057 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:18,059 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:18,084 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:19,064 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:20,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:21,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:22,055 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:23,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:23,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:24,064 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:25,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:26,068 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:27,065 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:28,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:28,965 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:29,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:30,059 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:30,598 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:44:30,955 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:30,955 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:31,062 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:32,066 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:33,065 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:34,050 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:34,060 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:35,063 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:36,062 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:37,078 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:38,075 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:39,080 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:39,968 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:40,078 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:41,079 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:42,081 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:43,079 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:44,081 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:44,972 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:45,079 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:45,948 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:44:45,948 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:44:46,083 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:47,084 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:48,083 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:49,084 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:50,087 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:50,099 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:51,089 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:52,091 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:53,092 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:54,089 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:55,092 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:55,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:44:56,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:57,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:58,093 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:44:59,097 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:00,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:00,599 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:45:00,949 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:00,949 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:01,036 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:01,099 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:02,098 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:03,099 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:04,102 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:05,100 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:06,103 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:06,967 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:07,103 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:08,112 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:09,112 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:10,111 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:11,117 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:12,116 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:12,967 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:13,121 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:14,119 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:15,120 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:15,953 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:15,953 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:16,128 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:17,121 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:18,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:18,122 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:19,123 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:20,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:21,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:22,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:23,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:23,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:24,125 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:25,131 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:26,124 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:27,130 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:28,129 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:29,131 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:29,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:30,126 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:30,599 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:45:30,952 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:30,952 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:31,129 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:32,127 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:33,133 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:34,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:35,044 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:35,132 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:36,134 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:37,134 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:38,136 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:39,136 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:40,137 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:40,972 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:41,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:42,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:43,135 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:44,138 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:45,144 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:45,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:45:45,958 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:45:46,069 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:46,142 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:47,138 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:48,146 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:49,148 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:50,148 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:51,151 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:51,974 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:52,145 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:53,151 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:54,152 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:55,150 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:56,151 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:56,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:45:57,154 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:58,152 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:45:59,152 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:00,158 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:00,599 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:46:00,955 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:00,956 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:01,154 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:02,088 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:02,154 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:03,157 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:04,158 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:05,155 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:06,158 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:07,162 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:07,977 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:08,157 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:09,160 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:10,160 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:11,162 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:12,165 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:12,983 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:13,166 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:14,168 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:15,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:15,964 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:15,964 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:16,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:17,172 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:18,052 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:18,167 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:19,167 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:20,166 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:21,168 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:22,169 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:23,176 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:23,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:24,172 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:25,170 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:26,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:27,174 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:28,171 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:29,173 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:29,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:30,178 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:30,600 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:46:30,959 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:30,960 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:31,177 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:32,181 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:33,177 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:34,178 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:35,078 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:35,184 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:36,186 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:37,185 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:38,186 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:39,190 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:40,191 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:40,981 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:41,187 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:42,191 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:43,193 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:44,189 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:45,191 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:45,955 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:46:45,955 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:46:46,107 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:46,188 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:47,189 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:48,192 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:49,193 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:50,197 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:51,206 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:51,989 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:52,196 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:53,203 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:54,202 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:55,204 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:56,199 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:57,202 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:57,988 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:46:58,203 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:46:59,207 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:00,207 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:00,600 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:47:00,958 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:00,958 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:01,202 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:02,208 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:03,063 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:03,208 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:04,208 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:05,206 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:06,205 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:07,209 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:08,210 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:08,995 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:09,215 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:10,210 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:11,210 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:12,211 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:13,220 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:13,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:14,219 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:15,215 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:15,957 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:15,957 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:16,218 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:17,219 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:18,226 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:19,026 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:19,222 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:20,223 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:21,221 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:22,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:23,227 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:24,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:24,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:25,225 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:26,226 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:27,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:28,227 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:29,224 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:29,998 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:30,225 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:30,601 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:47:30,965 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:30,965 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:31,225 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:32,233 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:33,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:34,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:35,111 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:35,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:36,233 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:37,234 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:38,235 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:39,236 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:40,236 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:40,996 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:41,234 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:42,241 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:43,237 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:44,238 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:45,239 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:45,964 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:47:45,964 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:47:46,038 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:46,244 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:47,239 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:48,240 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:49,240 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:50,246 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:51,241 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:52,002 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:52,248 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:53,245 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:54,250 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:55,255 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:56,246 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:57,249 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:57,999 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:47:58,247 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:47:59,247 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:00,249 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:00,601 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:48:00,961 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:00,961 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:01,248 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:02,249 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:03,066 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:03,253 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:04,255 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:05,251 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:06,250 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:07,254 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:08,251 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:09,004 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:09,254 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:10,255 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:11,253 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:12,253 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:13,262 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:14,011 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:14,257 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:15,256 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:15,966 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:15,966 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:16,261 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:17,260 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:18,261 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:19,116 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:19,264 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:20,271 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:21,265 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:22,267 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:23,272 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:24,270 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:25,006 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:25,268 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:26,276 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:27,271 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:28,268 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:29,272 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:30,009 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:30,271 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:30,602 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:48:30,978 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:30,978 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:31,273 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:32,272 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:33,273 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:34,275 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:35,108 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:35,275 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:36,284 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:37,286 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:38,290 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:39,286 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:40,291 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:41,013 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:41,290 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:42,292 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:43,294 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:44,298 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:45,295 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:45,972 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:48:45,972 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:48:46,119 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:46,299 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:47,304 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:48,297 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:49,297 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:50,299 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:51,311 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:52,007 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:52,306 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:53,300 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:54,304 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:55,306 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:56,305 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:57,013 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:48:57,302 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:58,303 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:48:59,310 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:00,302 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:00,602 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:49:00,968 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:00,969 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:01,305 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:02,082 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:02,305 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:03,307 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:04,307 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:05,306 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:06,307 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:07,308 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:08,021 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:08,309 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:09,308 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:10,312 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:11,315 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:12,316 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:13,315 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:14,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:14,311 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:15,310 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:15,973 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:15,973 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:16,316 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:17,316 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:18,315 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:19,070 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:19,318 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:20,319 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:21,317 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:22,320 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:23,318 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:24,320 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:25,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:25,322 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:26,322 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:27,322 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:28,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:29,323 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:29,635 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:49:29,635 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:49:29,636 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:49:29,679 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:49:29,679 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:49:29,932 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/iuh8sx1j-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:49:30,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:30,401 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:30,603 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:49:30,970 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:30,970 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:31,329 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:31,679 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:32,323 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:33,323 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:33,680 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:34,326 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:35,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:35,681 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:35,826 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:36,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:37,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:38,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:39,324 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:40,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:41,010 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:41,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:42,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:43,325 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:44,327 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:45,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:45,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:49:45,969 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:49:46,061 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:46,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:47,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:48,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:49,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:49,686 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:50,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:51,328 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:51,686 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:49:52,010 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:52,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:53,330 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:54,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:55,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:56,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:57,011 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:49:57,332 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:58,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:49:59,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:00,331 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:00,603 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:50:00,969 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:00,970 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:01,332 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:02,321 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:02,333 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:02,873 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:02,873 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:02,873 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 with policy now +2025-01-04 22:50:02,874 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:02,874 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:02,874 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:03,093 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/qyjas31q-media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:50:03,333 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:03,691 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:03,691 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:03,692 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:50:03,692 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:50:03,973 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/7s0lyx8z-media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:50:04,336 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:04,692 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:05,337 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:06,348 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:07,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:07,693 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:08,161 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:08,344 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:09,340 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:09,693 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:10,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:11,353 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:12,346 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:13,348 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:14,020 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:14,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:14,690 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:14,690 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:14,690 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:14,695 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:14,695 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:15,346 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:15,696 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:15,976 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:15,976 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:16,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:17,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:17,696 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:18,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:19,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:19,693 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:19,697 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:20,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:21,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:22,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:23,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:24,349 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:25,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:25,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:26,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:27,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:27,700 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:28,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:29,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:29,700 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:30,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:30,603 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:50:30,605 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:30,970 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:30,970 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:31,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:32,350 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:33,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:34,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:35,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:35,702 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:36,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:36,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:37,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:37,703 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:38,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:39,351 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:39,704 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:40,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:41,016 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:41,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:41,705 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:42,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:43,353 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:43,706 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:44,353 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:45,352 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:45,979 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:50:45,980 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:50:46,027 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:46,360 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:46,725 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:46,725 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 with policy now +2025-01-04 22:50:46,727 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:46,728 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:46,728 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:46,728 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:46,948 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/qdd3b483-media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:50:47,362 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:47,707 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:47,707 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:47,707 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:50:47,708 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval +2025-01-04 22:50:47,988 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/0imva0y3-media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:50:48,380 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:49,360 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:50,360 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:51,367 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:51,708 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:52,023 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:52,358 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:53,365 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:53,709 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:54,369 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:55,378 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:56,375 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:57,025 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:50:57,372 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:57,481 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:50:57,482 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:50:57,687 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval +2025-01-04 22:50:57,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table +2025-01-04 22:50:57,907 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/wr0kp33z-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:50:57,934 INFO wandb-upload_2:2040553 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6fqhwqg_ +2025-01-04 22:50:57,986 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/g4jl9fke-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:58,133 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:50:58,142 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:50:58,327 INFO SenderThread:2040553 [sender.py:send_request_log_artifact():1525] logged artifact run-u0i99mq0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIwNjY3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NTMzMA==', 'latestArtifact': None}} +2025-01-04 22:50:58,328 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:58,328 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-04 22:50:58,328 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:58,328 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,329 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,329 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:58,329 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,329 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,330 DEBUG SenderThread:2040553 [sender.py:send():379] send: telemetry +2025-01-04 22:50:58,330 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,331 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,331 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,332 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,332 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,332 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,333 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,333 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,333 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,333 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,334 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,334 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,334 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,334 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,335 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,335 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,335 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,335 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,335 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,336 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,336 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:58,336 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:58,336 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:50:58,365 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:58,550 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/z2rvkuk3-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:50:58,677 INFO wandb-upload_1:2040553 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwcd1hnub +2025-01-04 22:50:58,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:58,711 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:50:58,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:50:58,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval +2025-01-04 22:50:59,063 INFO SenderThread:2040553 [sender.py:send_request_log_artifact():1525] logged artifact run-u0i99mq0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIwNjY3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NTMzMQ==', 'latestArtifact': None}} +2025-01-04 22:50:59,063 DEBUG SenderThread:2040553 [sender.py:send():379] send: files +2025-01-04 22:50:59,063 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json with policy now +2025-01-04 22:50:59,064 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:50:59,064 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:50:59,064 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:50:59,247 INFO wandb-upload_2:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/cjn02b2n-media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:50:59,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:50:59,476 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/6jeypiub-media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:50:59,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:50:59,712 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:00,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:00,604 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:51:00,971 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:51:00,972 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:51:01,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:02,095 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:02,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:03,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:04,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:05,366 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:06,367 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:07,367 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:08,022 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:08,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:09,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:10,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:11,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:12,368 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:12,697 DEBUG SenderThread:2040553 [sender.py:send():379] send: exit +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_exit():588] handling runtime: 642 +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_exit():594] send defer +2025-01-04 22:51:12,698 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,698 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:51:12,698 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:51:12,698 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:51:12,699 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,699 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:51:12,699 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,699 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:51:12,699 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:51:12,699 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,699 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:51:12,699 INFO HandlerThread:2040553 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:51:12,699 DEBUG SystemMonitor:2040553 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:51:12,699 DEBUG SystemMonitor:2040553 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:51:12,700 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:51:12,700 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:51:12,717 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:51:12,785 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:51:12,785 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:51:12,785 INFO HandlerThread:2040553 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:51:12,786 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,786 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:51:12,786 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:51:12,786 DEBUG SenderThread:2040553 [sender.py:send():379] send: stats +2025-01-04 22:51:12,786 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,786 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:51:12,786 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:12,786 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:51:12,787 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:51:12,787 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:12,787 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:51:13,718 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:51:19,747 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:19,748 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,748 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:51:19,748 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:51:19,748 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:19,748 DEBUG SenderThread:2040553 [sender.py:send():379] send: history +2025-01-04 22:51:19,749 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:51:19,749 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:19,749 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:51:19,750 DEBUG SenderThread:2040553 [sender.py:send():379] send: summary +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:51:19,750 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:51:19,750 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:51:19,751 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:19,751 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:51:19,751 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,751 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:51:19,754 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:51:19,855 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:51:19,856 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:19,856 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:51:19,856 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:19,856 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:51:20,699 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:20,720 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:51:20,721 INFO Thread-12 :2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml +2025-01-04 22:51:21,862 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:51:21,862 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:21,863 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:21,863 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:51:21,863 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:21,863 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:51:21,863 INFO SenderThread:2040553 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:51:21,865 INFO SenderThread:2040553 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:51:21,867 INFO SenderThread:2040553 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:51:21,882 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:51:21,882 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:21,882 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:51:21,882 DEBUG SenderThread:2040553 [sender.py:send():379] send: artifact +2025-01-04 22:51:22,062 INFO SenderThread:2040553 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:51:22,062 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:22,062 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:51:22,062 INFO SenderThread:2040553 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:51:22,700 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml config.yaml +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json wandb-summary.json +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt requirements.txt +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:51:22,722 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log output.log +2025-01-04 22:51:22,723 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/diff.patch diff.patch +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 media/videos/eval/4_eval_4_d212573bac6676e018bd.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 media/videos/eval/2_eval_2_06d0e3b5afe6377da038.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/0_eval_0_64740432aabccb229691.mp4 media/videos/eval/0_eval_0_64740432aabccb229691.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 media/videos/eval/1_eval_1_acff89a56e2528ff5ad3.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 media/videos/eval/3_eval_3_ee5c1dd7abeb860c9608.mp4 +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json media/table/eval/total_success_rate_ci_6_22e3aa8cab918fc471a4.table.json +2025-01-04 22:51:22,728 INFO SenderThread:2040553 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:51:22,729 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:51:22,729 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:22,731 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:22,732 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:51:22,733 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:22,733 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:51:22,733 INFO SenderThread:2040553 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:51:22,904 INFO wandb-upload_2:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/config.yaml +2025-01-04 22:51:22,906 INFO wandb-upload_0:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/wandb-summary.json +2025-01-04 22:51:22,949 INFO wandb-upload_1:2040553 [upload_job.py:push():130] Uploaded file /tmp/tmpmaezpzxiwandb/osc069yn-events.out.tfevents.1736048430.leonmkim-ROG-Strix-G15CS-G15CS.2040388.0 +2025-01-04 22:51:22,957 INFO wandb-upload_3:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/requirements.txt +2025-01-04 22:51:22,985 INFO wandb-upload_4:2040553 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/files/output.log +2025-01-04 22:51:23,186 INFO Thread-11 (_thread_body):2040553 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:51:23,186 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,186 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:51:23,186 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,186 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:51:23,187 INFO SenderThread:2040553 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:51:23,188 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:51:23,188 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,188 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:51:23,188 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,188 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:51:23,188 INFO SenderThread:2040553 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:51:23,479 INFO SenderThread:2040553 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:51:23,479 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:51:23,479 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,479 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,480 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:51:23,480 INFO SenderThread:2040553 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:51:23,480 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:51:23,480 INFO HandlerThread:2040553 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send():379] send: final +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send():379] send: footer +2025-01-04 22:51:23,480 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: defer +2025-01-04 22:51:23,480 INFO SenderThread:2040553 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:51:23,481 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:23,481 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:23,481 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:51:23,482 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:51:23,482 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:51:23,483 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:51:23,484 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:51:23,484 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:51:23,565 DEBUG SenderThread:2040553 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:51:23,566 INFO MainThread:2040553 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:51:23,566 INFO MainThread:2040553 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:51:23,566 INFO MainThread:2040553 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:51:23,567 DEBUG HandlerThread:2040553 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:51:23,567 INFO HandlerThread:2040553 [handler.py:finish():882] shutting down handler +2025-01-04 22:51:24,482 INFO WriterThread:2040553 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb +2025-01-04 22:51:24,566 INFO SenderThread:2040553 [sender.py:finish():1615] shutting down sender +2025-01-04 22:51:24,566 INFO SenderThread:2040553 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:51:24,566 INFO SenderThread:2040553 [file_pusher.py:join():175] waiting for file pusher diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug.log b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..76224a41211001deb425e06406307614344fb772 --- /dev/null +++ b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Configure stats pid to 2040388 +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/settings +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:40:30,226 INFO MainThread:2040388 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug.log +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/logs/debug-internal.log +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0'} +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():619] starting backend +2025-01-04 22:40:30,227 INFO MainThread:2040388 [wandb_init.py:init():623] setting up manager +2025-01-04 22:40:30,230 INFO MainThread:2040388 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:40:30,231 INFO MainThread:2040388 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:40:30,244 INFO MainThread:2040388 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:40:30,251 INFO MainThread:2040388 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:40:30,510 INFO MainThread:2040388 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:40:30,581 INFO MainThread:2040388 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:40:30,581 INFO MainThread:2040388 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:40:30,936 INFO MainThread:2040388 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:40:30,936 INFO MainThread:2040388 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:40:30,937 INFO MainThread:2040388 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:40:30,937 INFO MainThread:2040388 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:40:30,938 INFO MainThread:2040388 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:40:30,938 INFO MainThread:2040388 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/u0i99mq0/tb, True +2025-01-04 22:40:35,416 INFO MainThread:2040388 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:51:24,567 WARNING MsgRouterThr:2040388 [router.py:message_loop():77] message_loop has been closed diff --git a/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e3c713ede2b0e01479f5622a3b8c96bf7322c813 Binary files /dev/null and b/u0i99mq0/wandb/run-20250104_224030-u0i99mq0/run-u0i99mq0.wandb differ diff --git a/yxg6asez/.hydra/config.yaml b/yxg6asez/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/yxg6asez/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/yxg6asez/.hydra/hydra.yaml b/yxg6asez/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb12a11c5034b4ac2bb935db664c1d170f715c7 --- /dev/null +++ b/yxg6asez/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/yxg6asez/.hydra/overrides.yaml b/yxg6asez/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/yxg6asez/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/yxg6asez/episode_rosbags/aligned_depth_to_color_K.npy b/yxg6asez/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yxg6asez/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yxg6asez/episode_rosbags/cam_tf_world.npy b/yxg6asez/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/yxg6asez/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/yxg6asez/episode_rosbags/color_K.npy b/yxg6asez/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yxg6asez/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yxg6asez/episode_rosbags/depth_K.npy b/yxg6asez/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/yxg6asez/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/yxg6asez/episode_rosbags/episode_1_2025-01-04-20-49-45.bag b/yxg6asez/episode_rosbags/episode_1_2025-01-04-20-49-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..1d6911679bdc7b44cdda9f57ce0ceca453968494 --- /dev/null +++ b/yxg6asez/episode_rosbags/episode_1_2025-01-04-20-49-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34cb0261a529fdf64f057b82a6d299729e35b6ed3c13f15c3f3e79bc35f720fa +size 1411528956 diff --git a/yxg6asez/eval_robot.log b/yxg6asez/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bb3966a60d7d7c2c85f30f621665cc4605bb55cb --- /dev/null +++ b/yxg6asez/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 20:48:41,742][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:48:41,745][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/yxg6asez/eval_video/0_eval.mp4 b/yxg6asez/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ec61ece36c7f460c909b8fd0d434d4213d0c2834 --- /dev/null +++ b/yxg6asez/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83e67f40a4f92c94379976125df2bdad392295c6f7073a4cf69cb7fe47514fb +size 1197514 diff --git a/yxg6asez/eval_video/1_eval.mp4 b/yxg6asez/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ffa75342d3b1a1b30a119bf6ea81b80131871b5 --- /dev/null +++ b/yxg6asez/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:357b7ef1e9cc666b9c495912ca611b94d5db5dda02ed9d6b260bfb9b2d1e4fc6 +size 1138816 diff --git a/yxg6asez/eval_video/2_eval.mp4 b/yxg6asez/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a1f4bd61b9ea96621fe0a31003e70e35d9f07e08 --- /dev/null +++ b/yxg6asez/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:986a953e86ee8af4afd22bdbbb47b978b6f74b94d7632ef1ffeb2971c338ba6b +size 1135436 diff --git a/yxg6asez/eval_video/3_eval.mp4 b/yxg6asez/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8266bf378ba4cf7c5f2b19505c3b2a3664867e65 --- /dev/null +++ b/yxg6asez/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4948f3833cf9d19271a47a51e8afbde4478eda8a0c16c35e24549b0e7f58f95 +size 1133329 diff --git a/yxg6asez/eval_video/4_eval.mp4 b/yxg6asez/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b9cb0b18ba73bf96352d6332757e1baab35dda2 --- /dev/null +++ b/yxg6asez/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f660567809dcf580f6258eed7f7cc1b26cbaec1b24e5b4b2365bc346fe3fd7 +size 1146609 diff --git a/yxg6asez/tb/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 b/yxg6asez/tb/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 new file mode 100644 index 0000000000000000000000000000000000000000..116a40b5ba4e992bd0a126bf6e08a2fe39dd5864 --- /dev/null +++ b/yxg6asez/tb/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93448773424e3a0850718f7cc685aa9d32ce81571807e6276f4347a40a8be9f6 +size 6539 diff --git a/yxg6asez/wandb/debug-internal.log b/yxg6asez/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..53dd8f9d90857cdb921d98e6c67b0f566fa6a38a --- /dev/null +++ b/yxg6asez/wandb/debug-internal.log @@ -0,0 +1,740 @@ +2025-01-04 20:48:47,522 INFO StreamThr :2017702 [internal.py:wandb_internal():85] W&B internal server running at pid: 2017702, started at: 2025-01-04 20:48:47.522669 +2025-01-04 20:48:47,524 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:48:47,536 INFO WriterThread:2017702 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb +2025-01-04 20:48:47,537 DEBUG SenderThread:2017702 [sender.py:send():379] send: header +2025-01-04 20:48:47,544 DEBUG SenderThread:2017702 [sender.py:send():379] send: run +2025-01-04 20:48:47,753 INFO SenderThread:2017702 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files +2025-01-04 20:48:47,753 INFO SenderThread:2017702 [sender.py:_start_run_threads():1188] run started: yxg6asez with start time 1736041727.523208 +2025-01-04 20:48:47,769 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:48:47,770 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:48:47,815 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:48:47,820 DEBUG HandlerThread:2017702 [system_info.py:__init__():26] System info init +2025-01-04 20:48:47,820 DEBUG HandlerThread:2017702 [system_info.py:__init__():41] System info init done +2025-01-04 20:48:47,820 INFO HandlerThread:2017702 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:48:47,820 INFO SystemMonitor:2017702 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:48:47,820 INFO HandlerThread:2017702 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:48:47,821 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:48:47,821 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:48:47,821 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:48:47,822 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:48:47,822 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:48:47,833 DEBUG HandlerThread:2017702 [system_info.py:probe():152] Probing system +2025-01-04 20:48:47,835 DEBUG HandlerThread:2017702 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_info.py:probe():200] Probing system done +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:48:47.833575', 'startedAt': '2025-01-05T01:48:47.511040', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.498, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.986, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 623.8496360778809}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:48:47,839 INFO HandlerThread:2017702 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:48:47,839 INFO HandlerThread:2017702 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_info.py:_save_code():44] Saving code +2025-01-04 20:48:47,845 DEBUG HandlerThread:2017702 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:48:47,845 DEBUG HandlerThread:2017702 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:48:47,907 DEBUG HandlerThread:2017702 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:48:47,907 INFO HandlerThread:2017702 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:48:47,911 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:48:47,911 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:48:47,911 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:48:47,911 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:48:48,139 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/soy2expn-wandb-metadata.json +2025-01-04 20:48:48,156 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:48:48,156 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:48:48,156 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:48:48,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:48:48,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:48,159 INFO HandlerThread:2017702 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/tb" + save: true +} +control { + relay_id: "5a5d79f676e44e83a90af2b90086f382" +} +_info { + stream_id: "yxg6asez" +} + +2025-01-04 20:48:48,159 DEBUG HandlerThread:2017702 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:48:48,160 INFO wandb-upload_1:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/irvppdhm-code/FISH/eval_robot.py +2025-01-04 20:48:48,201 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/vpqvyavf-diff.patch +2025-01-04 20:48:48,311 DEBUG SenderThread:2017702 [sender.py:send():379] send: telemetry +2025-01-04 20:48:48,312 DEBUG SenderThread:2017702 [sender.py:send():379] send: tbrecord +2025-01-04 20:48:48,312 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:48:48,312 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 with policy live +2025-01-04 20:48:48,495 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/1mydel46-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:48:48,754 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH +2025-01-04 20:48:49,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:50,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:50,754 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:51,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:51,410 DEBUG SenderThread:2017702 [sender.py:send():379] send: config +2025-01-04 20:48:52,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:52,754 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:53,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:53,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:54,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:54,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:55,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:56,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:56,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:57,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:58,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:58,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:58,756 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:59,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:00,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:00,757 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:01,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:02,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:03,156 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:03,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:03,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:03,292 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:04,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:04,758 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:05,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:06,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:06,758 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:07,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:08,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:08,759 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:09,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:09,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:10,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:11,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:12,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:13,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:14,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:14,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:15,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:16,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:17,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:18,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:18,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:18,197 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:19,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:19,228 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:19,762 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml +2025-01-04 20:49:20,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:21,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:22,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:22,763 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:23,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:24,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:25,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:25,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:26,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:27,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:28,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:29,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:30,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:30,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:31,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:32,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:33,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:33,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:33,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:33,807 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:49:33,807 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 with policy now +2025-01-04 20:49:33,807 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:49:33,808 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:49:33,808 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:49:33,809 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:49:34,073 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/5nvdvrdg-media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:49:34,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media +2025-01-04 20:49:35,070 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/ayinxzcn-media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:49:35,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:35,814 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:36,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:37,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:38,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:38,767 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:39,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:40,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:41,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:41,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:42,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:42,769 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:49:42,769 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:43,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:44,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:44,769 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:45,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:46,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:46,770 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:47,057 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:47,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:47,822 DEBUG SystemMonitor:2017702 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:49:47,824 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:49:48,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:48,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:48,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:48,771 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:49,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:50,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:51,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:52,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:52,248 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:53,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:54,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:55,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:56,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:57,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:58,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:58,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:59,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:00,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:01,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:02,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:03,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:03,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:03,205 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:03,296 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:04,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:05,168 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:06,168 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:07,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:08,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:09,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:09,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:10,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:10,779 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:11,170 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:12,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:13,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:13,924 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:50:13,924 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:50:13,924 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 with policy now +2025-01-04 20:50:13,924 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:50:13,925 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:50:13,925 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:50:14,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:14,184 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/95r3erdc-media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:50:14,946 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:15,074 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/oiksaks6-media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:50:15,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:16,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:16,781 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:17,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:17,824 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:50:18,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:18,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:18,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:18,781 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:19,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:20,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:20,318 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:21,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:22,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:22,783 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:23,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:23,783 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:50:24,011 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/fgzgymdz-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:50:24,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:24,548 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:50:24,548 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:50:24,548 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:50:24,784 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:50:24,784 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:25,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:25,805 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:26,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:26,784 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:27,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:28,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:28,785 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:29,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:30,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:31,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:31,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:32,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:33,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:33,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:33,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:34,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:35,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:36,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:36,247 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:37,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:38,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:39,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:40,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:41,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:42,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:42,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:43,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:44,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:45,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:46,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:47,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:47,825 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:50:47,825 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:48,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:48,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:48,205 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:49,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:50,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:51,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:52,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:53,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:53,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:54,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:55,179 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:55,698 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:50:55,698 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 with policy now +2025-01-04 20:50:55,698 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:50:55,699 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:50:55,699 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:50:55,700 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:50:55,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:50:55,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:50:55,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:50:55,958 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/gt7vrsj8-media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:50:56,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:56,208 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/shevj0e6-media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:50:56,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:57,180 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:57,795 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:58,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:58,179 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:59,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:00,182 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:00,796 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:01,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:02,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:03,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:03,159 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:03,183 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:03,290 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:04,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:04,797 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:05,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:05,377 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:51:05,377 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:51:05,377 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:51:05,797 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:51:05,797 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:51:06,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:06,798 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:07,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:08,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:08,747 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:09,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:10,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:10,799 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:11,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:12,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:13,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:14,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:14,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:15,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:16,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:17,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:17,825 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:51:18,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:18,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:18,205 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:19,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:19,277 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:20,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:21,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:22,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:23,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:24,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:25,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:25,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:26,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:27,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:28,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:29,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:30,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:30,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:31,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:32,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:33,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:33,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:33,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:34,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:35,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:35,275 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:36,194 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:36,807 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:37,005 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:51:37,005 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 with policy now +2025-01-04 20:51:37,006 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:51:37,007 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:51:37,007 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:51:37,007 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:51:37,190 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:37,263 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/v9uciaj8-media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:51:37,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:51:37,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:51:37,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:51:38,087 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/sq3dspj5-media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:51:38,192 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:38,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:39,202 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:40,199 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:40,741 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:41,196 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:42,198 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:42,810 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:43,198 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:44,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:44,810 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:45,206 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:46,214 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:46,566 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:47,209 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:47,826 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:51:48,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:48,160 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:48,209 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:49,209 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:50,212 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:51,210 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:52,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:52,268 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:53,216 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:54,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:55,215 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:56,215 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:57,217 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:58,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:58,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:59,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:00,214 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:01,218 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:02,216 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:03,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:03,166 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:03,216 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:03,288 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:04,217 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:05,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:06,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:06,448 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:06,448 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:06,449 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:06,817 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:06,818 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:07,056 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/js493f6e-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:07,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:08,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:08,818 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:09,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:09,220 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:10,257 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:11,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:12,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:12,820 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:13,220 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:14,220 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:15,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:15,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:16,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:17,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:17,826 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:52:18,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:18,165 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:18,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:19,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:20,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:20,303 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:20,822 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:21,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:22,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:22,823 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:23,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:24,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:24,824 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:25,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:26,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:26,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:27,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:28,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:29,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:30,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:31,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:31,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:32,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:33,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:33,159 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:33,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:34,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:34,828 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:35,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:36,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:36,881 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:37,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:38,235 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:38,399 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:52:38,399 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 with policy now +2025-01-04 20:52:38,402 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:52:38,402 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:38,402 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:38,403 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:38,668 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/dh8p73ib-media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:52:38,829 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:38,829 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:38,830 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:52:38,830 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:52:39,103 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/46c7v8ay-media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:52:39,237 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:40,238 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:40,830 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:41,242 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:42,130 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:42,233 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:43,238 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:44,236 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:44,831 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:45,235 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:46,239 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:46,832 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:47,245 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:47,827 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:52:47,827 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:48,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:48,171 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:48,249 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:48,354 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:52:48,354 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:52:48,578 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:52:48,833 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:48,833 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval +2025-01-04 20:52:48,839 INFO wandb-upload_2:2017702 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl940nc8n +2025-01-04 20:52:48,957 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:52:48,965 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:52:49,025 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/fvlqkexr-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:52:49,242 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:49,286 INFO SenderThread:2017702 [sender.py:send_request_log_artifact():1525] logged artifact run-yxg6asez-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMDkzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTM3NA==', 'latestArtifact': None}} +2025-01-04 20:52:49,286 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:52:49,286 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 20:52:49,286 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:49,286 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,287 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,287 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:49,287 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,287 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,288 DEBUG SenderThread:2017702 [sender.py:send():379] send: telemetry +2025-01-04 20:52:49,288 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,288 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,288 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,288 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,290 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,290 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,290 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,291 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,291 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,291 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,291 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,292 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,292 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,292 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,292 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,293 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,293 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,293 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,293 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,293 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,294 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,294 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,294 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:52:49,525 INFO wandb-upload_1:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/9tq3l4yd-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:52:49,675 INFO wandb-upload_0:2017702 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp24hn42wx +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval +2025-01-04 20:52:50,047 INFO SenderThread:2017702 [sender.py:send_request_log_artifact():1525] logged artifact run-yxg6asez-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMDk1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTM3Ng==', 'latestArtifact': None}} +2025-01-04 20:52:50,047 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:52:50,047 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json with policy now +2025-01-04 20:52:50,048 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:50,048 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:50,048 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:50,220 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/8ngkshr1-media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:52:50,242 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:50,297 DEBUG SenderThread:2017702 [sender.py:send():379] send: exit +2025-01-04 20:52:50,297 INFO SenderThread:2017702 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:send_exit():588] handling runtime: 242 +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:send_exit():594] send defer +2025-01-04 20:52:50,298 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,298 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:52:50,298 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:52:50,299 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:52:50,299 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,299 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:52:50,299 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,299 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:52:50,299 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:52:50,299 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,299 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:52:50,299 INFO HandlerThread:2017702 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:52:50,300 DEBUG SystemMonitor:2017702 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:52:50,300 DEBUG SystemMonitor:2017702 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:52:50,301 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:52:50,301 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:52:50,353 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:52:50,353 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:52:50,353 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:52:50,353 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,353 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:52:50,353 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:52:50,353 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:52:50,354 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,354 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:52:50,354 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,354 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:52:50,354 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:52:50,354 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,354 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:52:50,459 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/0msw9jif-media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:52:50,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:50,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:52,835 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:56,511 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:52:56,511 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,512 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:56,512 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:52:56,512 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:52:56,512 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:52:56,513 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:52:56,513 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:56,513 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:52:56,514 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:56,514 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:56,515 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:56,515 DEBUG SenderThread:2017702 [sender.py:send():379] send: summary +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:56,519 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:52:56,519 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:56,519 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:52:56,519 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:52:56,523 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:56,667 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:52:56,668 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:56,668 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:52:56,668 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,668 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:52:56,837 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:56,837 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml +2025-01-04 20:52:57,299 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:52:59,723 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:52:59,723 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:52:59,724 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:59,724 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:52:59,724 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:59,724 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:52:59,724 INFO SenderThread:2017702 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:52:59,726 INFO SenderThread:2017702 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:52:59,728 INFO SenderThread:2017702 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:52:59,736 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:52:59,737 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:59,737 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:52:59,737 DEBUG SenderThread:2017702 [sender.py:send():379] send: artifact +2025-01-04 20:52:59,838 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:53:00,192 INFO wandb-upload_0:2017702 [upload_job.py:push():85] Skipped uploading /tmp/tmp24i4tet4/wandb-job.json +2025-01-04 20:53:00,192 INFO wandb-upload_1:2017702 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphf6br093 +2025-01-04 20:53:00,299 INFO wandb-upload_2:2017702 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppgbbl2ol +2025-01-04 20:53:00,302 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:00,885 INFO SenderThread:2017702 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'versionIndex': 417}}} +2025-01-04 20:53:00,885 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:00,885 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:53:00,885 INFO SenderThread:2017702 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:53:01,839 INFO SenderThread:2017702 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files +2025-01-04 20:53:01,839 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml config.yaml +2025-01-04 20:53:01,839 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json wandb-summary.json +2025-01-04 20:53:01,840 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt requirements.txt +2025-01-04 20:53:01,840 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:53:01,840 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:53:01,844 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log output.log +2025-01-04 20:53:01,844 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch diff.patch +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:53:01,846 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:53:01,846 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:01,847 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:01,848 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:53:01,849 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:01,849 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:53:01,849 INFO SenderThread:2017702 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:53:02,015 INFO wandb-upload_1:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:53:02,020 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt +2025-01-04 20:53:02,035 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml +2025-01-04 20:53:02,063 INFO wandb-upload_4:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:53:02,116 INFO wandb-upload_3:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/edn2qdfq-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:53:02,302 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:02,302 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:02,316 INFO Thread-11 (_thread_body):2017702 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:53:02,316 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,316 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:53:02,317 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,317 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:53:02,317 INFO SenderThread:2017702 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:53:02,318 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:53:02,318 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,318 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:53:02,318 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,318 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:53:02,318 INFO SenderThread:2017702 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:53:02,487 INFO SenderThread:2017702 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:53:02,487 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:53:02,488 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,488 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,488 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:53:02,488 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:53:02,488 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,488 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send():379] send: final +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send():379] send: footer +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,489 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:53:02,489 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:02,489 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:02,489 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:53:02,490 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:02,490 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:53:02,490 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:53:02,491 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:02,491 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:53:02,551 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:02,551 INFO MainThread:2017702 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:53:02,552 INFO MainThread:2017702 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:53:02,552 INFO MainThread:2017702 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:53:02,552 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:53:02,552 INFO HandlerThread:2017702 [handler.py:finish():882] shutting down handler +2025-01-04 20:53:03,490 INFO WriterThread:2017702 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb +2025-01-04 20:53:03,551 INFO SenderThread:2017702 [sender.py:finish():1615] shutting down sender +2025-01-04 20:53:03,551 INFO SenderThread:2017702 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:53:03,552 INFO SenderThread:2017702 [file_pusher.py:join():175] waiting for file pusher diff --git a/yxg6asez/wandb/debug.log b/yxg6asez/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5ccfecb5f058f0d69d6ddd351bee4eb46b868864 --- /dev/null +++ b/yxg6asez/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Configure stats pid to 2017514 +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/settings +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug.log +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug-internal.log +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez'} +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():619] starting backend +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():623] setting up manager +2025-01-04 20:48:47,521 INFO MainThread:2017514 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:48:47,523 INFO MainThread:2017514 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:48:47,536 INFO MainThread:2017514 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:48:47,542 INFO MainThread:2017514 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:48:47,769 INFO MainThread:2017514 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:48:47,810 INFO MainThread:2017514 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:48:47,810 INFO MainThread:2017514 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:48:48,159 INFO MainThread:2017514 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:48:48,159 INFO MainThread:2017514 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/tb, True +2025-01-04 20:48:51,409 INFO MainThread:2017514 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:53:03,553 WARNING MsgRouterThr:2017514 [router.py:message_loop():77] message_loop has been closed diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..faa77c13527fe860eb8dfe5daec33942fcea3134 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736041727 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9fa2cd2b55d1ee76eb00a71ab4767356dec5ccae --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[33, 35, 0.9428571428571428, 0.7908204197301967, 0.9943205228949389, 0.8324342145850978, 0.9927833619473274, 0.853344149816893, 0.9836125186608592, 0.8608765340619744, 0.983629776999486]]} \ No newline at end of file diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ec61ece36c7f460c909b8fd0d434d4213d0c2834 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83e67f40a4f92c94379976125df2bdad392295c6f7073a4cf69cb7fe47514fb +size 1197514 diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ffa75342d3b1a1b30a119bf6ea81b80131871b5 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:357b7ef1e9cc666b9c495912ca611b94d5db5dda02ed9d6b260bfb9b2d1e4fc6 +size 1138816 diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a1f4bd61b9ea96621fe0a31003e70e35d9f07e08 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:986a953e86ee8af4afd22bdbbb47b978b6f74b94d7632ef1ffeb2971c338ba6b +size 1135436 diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8266bf378ba4cf7c5f2b19505c3b2a3664867e65 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4948f3833cf9d19271a47a51e8afbde4478eda8a0c16c35e24549b0e7f58f95 +size 1133329 diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b9cb0b18ba73bf96352d6332757e1baab35dda2 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f660567809dcf580f6258eed7f7cc1b26cbaec1b24e5b4b2365bc346fe3fd7 +size 1146609 diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d6a30d1693f71c0fa4f6a6f0b5c450087a414915 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736041731.410115]: resetting environment +[INFO] [1736041731.413188]: cleared current plan +[INFO] [1736041731.414086]: moving to home +[INFO] [1736041732.416639]: reached home +[INFO] [1736041732.418042]: reset action history +[INFO] [1736041735.126361]: environment reset +Starting episode 0 +[INFO] [1736041735.128857]: resetting environment +[INFO] [1736041735.133758]: cleared current plan +[INFO] [1736041735.134884]: moving to home +[INFO] [1736041736.137260]: reached home +[INFO] [1736041736.138638]: reset action history +[INFO] [1736041738.846106]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736041742.081552]: resetting environment +[INFO] [1736041742.086710]: cleared current plan +[INFO] [1736041742.088972]: moving to home +[INFO] [1736041743.092691]: reached home +[INFO] [1736041743.094684]: reset action history +[INFO] [1736041745.801444]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/episode_rosbags/episode_0_2025-01-04-20-49-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2017514.log +started roslaunch server http://158.130.50.37:34095/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2017823] +process[print_text-2]: started with pid [2017824] +process[rosbag_record-3]: started with pid [2017825] +started bagging! +For topic gripper_width: timestamp difference is 100301913 for nearest: 1736041759918238572 - target: 1736041759817936659 at idx 210 +[INFO] [1736041772.841550]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736041773.584491]: Stored episode 1. +[INFO] [1736041773.584726]: Saving video... +[INFO] [1736041773.794771]: Video saved! +Starting episode 1 +[INFO] [1736041773.808580]: resetting environment +[INFO] [1736041773.812492]: cleared current plan +[INFO] [1736041773.813562]: moving to home +[INFO] [1736041777.416027]: reached home +[INFO] [1736041777.416492]: reset action history +[INFO] [1736041780.126904]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041782.085845]: resetting environment +[INFO] [1736041782.091316]: cleared current plan +[INFO] [1736041782.091494]: moving to home +[INFO] [1736041783.093274]: reached home +[INFO] [1736041783.094346]: reset action history +[INFO] [1736041785.801283]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/episode_rosbags/episode_1_2025-01-04-20-49-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2017514.log +started roslaunch server http://158.130.50.37:39467/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2017936] +process[print_text-5]: started with pid [2017937] +process[rosbag_record-6]: started with pid [2017961] +started bagging! +For topic gripper_width: timestamp difference is 101748960 for nearest: 1736041807984905736 - target: 1736041807883156776 at idx 249 +[INFO] [1736041812.774263]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736041813.567099]: Stored episode 2. +[INFO] [1736041813.567752]: Saving video... +[INFO] [1736041813.904046]: Video saved! +Starting episode 2 +[INFO] [1736041813.926721]: resetting environment +[INFO] [1736041813.945616]: cleared current plan +[INFO] [1736041813.945877]: moving to home +[INFO] [1736041817.649233]: reached home +[INFO] [1736041817.650827]: reset action history +[INFO] [1736041820.425216]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041823.755925]: resetting environment +[INFO] [1736041823.758787]: cleared current plan +[INFO] [1736041823.758942]: moving to home +[INFO] [1736041824.760406]: reached home +[INFO] [1736041824.760864]: reset action history +[INFO] [1736041827.468440]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/episode_rosbags/episode_2_2025-01-04-20-50-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2017514.log +started roslaunch server http://158.130.50.37:46741/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2018071] +process[print_text-8]: started with pid [2018072] +process[rosbag_record-9]: started with pid [2018096] +started bagging! +[INFO] [1736041854.441870]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736041855.238622]: Stored episode 3. +[INFO] [1736041855.241260]: Saving video... +[INFO] [1736041855.660846]: Video saved! +Starting episode 3 +[INFO] [1736041855.699465]: resetting environment +[INFO] [1736041855.713616]: cleared current plan +[INFO] [1736041855.721315]: moving to home +[WARN] [1736041855.595049]: Plan exhausted +[WARN] [1736041855.628124]: Plan exhausted +[WARN] [1736041855.666975]: Plan exhausted +[WARN] [1736041855.707242]: Plan exhausted +[INFO] [1736041859.429351]: reached home +[INFO] [1736041859.432086]: reset action history +[INFO] [1736041862.210994]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041864.837215]: resetting environment +[INFO] [1736041864.852453]: cleared current plan +[INFO] [1736041864.855539]: moving to home +[INFO] [1736041865.859236]: reached home +[INFO] [1736041865.861131]: reset action history +[INFO] [1736041868.746026]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/episode_rosbags/episode_3_2025-01-04-20-51-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2017514.log +started roslaunch server http://158.130.50.37:42315/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2018206] +process[print_text-11]: started with pid [2018207] +process[rosbag_record-12]: started with pid [2018231] +started bagging! +[INFO] [1736041895.718855]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1736041896.543277]: Stored episode 4. +[INFO] [1736041896.548029]: Saving video... +[INFO] [1736041896.970685]: Video saved! +Starting episode 4 +[INFO] [1736041897.016567]: resetting environment +[INFO] [1736041897.036428]: cleared current plan +[INFO] [1736041897.036839]: moving to home +[INFO] [1736041900.740166]: reached home +[INFO] [1736041900.744084]: reset action history +[INFO] [1736041903.522884]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041926.463586]: resetting environment +[INFO] [1736041926.477046]: cleared current plan +[INFO] [1736041926.478358]: moving to home +[INFO] [1736041927.484610]: reached home +[INFO] [1736041927.488465]: reset action history +[INFO] [1736041930.211872]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/episode_rosbags/episode_4_2025-01-04-20-52-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2017514.log +started roslaunch server http://158.130.50.37:32849/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2018376] +process[print_text-14]: started with pid [2018377] +process[rosbag_record-15]: started with pid [2018378] +started bagging! +For topic gripper_width: timestamp difference is 100640651 for nearest: 1736041938118235942 - target: 1736041938017595291 at idx 112 +For topic gripper_width: timestamp difference is 105563972 for nearest: 1736041940518261049 - target: 1736041940412697077 at idx 146 +For topic gripper_width: timestamp difference is 108860106 for nearest: 1736041942118257613 - target: 1736041942009397507 at idx 166 +For topic gripper_width: timestamp difference is 119603852 for nearest: 1736041953818233231 - target: 1736041953698629379 at idx 249 +[INFO] [1736041957.370538]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736041958.118441]: Stored episode 5. +[INFO] [1736041958.121669]: Saving video... +[INFO] [1736041958.353616]: Video saved! +[INFO] [1736041958.401178]: resetting environment +[INFO] [1736041958.419313]: cleared current plan +[INFO] [1736041958.420604]: moving to home +[INFO] [1736041962.128506]: reached home +[INFO] [1736041962.133671]: reset action history +[INFO] [1736041964.915259]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a42bdd64fe2ea859999aee13215aa5f0090a7eee --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T01:48:47.833575", + "startedAt": "2025-01-05T01:48:47.511040", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.498, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.986, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 623.8496360778809 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..38c1cde25b713d6933ea2e2dfe5d94882bd0122c --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a83e67f40a4f92c94379976125df2bdad392295c6f7073a4cf69cb7fe47514fb", "size": 1197514, "path": "media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4"}, "global_step": 664, "_timestamp": 1736041968.86476, "_runtime": 241.34155201911926, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "357b7ef1e9cc666b9c495912ca611b94d5db5dda02ed9d6b260bfb9b2d1e4fc6", "size": 1138816, "path": "media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4"}, "eval/num_success": 5.0, "episode": 35.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "986a953e86ee8af4afd22bdbbb47b978b6f74b94d7632ef1ffeb2971c338ba6b", "size": 1135436, "path": "media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "e4948f3833cf9d19271a47a51e8afbde4478eda8a0c16c35e24549b0e7f58f95", "size": 1133329, "path": "media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "11f660567809dcf580f6258eed7f7cc1b26cbaec1b24e5b4b2365bc346fe3fd7", "size": 1146609, "path": "media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://4o0fzxltxrkv6ikn4fu0iid7qypnnmxfxpiygiolywm0dhg3myheg9ql7vjcewaohhhdqohi5u1czor25aur5gy4eiqo0o945gnxfos5ialmv8ukr28aukbzj986l31a/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://fe9va3aa0xwii60qvl7lk7h2y3u8cx1f9sjz1kooy7f7kbxi07g862lpd53bvu31pgc8y7tdbondkm9xmc793sre8ifn7xyo2zrg8zu1xkt1j6l1ds1kg9gamn8ka458:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 33, "total_num_episodes": 35, "total_success_rate": 0.9428571428571428, "total_success_rate_lb_0.01": 0.7908204197301967, "total_success_rate_ub_0.01": 0.9943205228949389, "total_success_rate_lb_0.025": 0.8324342145850978, "total_success_rate_ub_0.025": 0.9927833619473274, "total_success_rate_lb_0.05": 0.853344149816893, "total_success_rate_ub_0.05": 0.9836125186608592, "total_success_rate_lb_0.1": 0.8608765340619744, "total_success_rate_ub_0.1": 0.983629776999486, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "57d6a400313d85e9473a05490cfe4c780266e7729ef529c415f35922f1421937", "size": 518, "artifact_path": "wandb-client-artifact://nicq66spdjwxv25bkrzo8tqypwofxzywwc8gzf3n490xzyqxmu7h6ofc1ix9slssgc16tesms0nsyx46ndsazigqbkw8apdpvj595l8jrz8fq9cjjmyvmjxmm4rkw5cw/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://v57ojhbp6d1ukh27xzahtxwwj863wm1tx58t3mfzr7c3va18q27ut0jxj6ufa7yr5qyoxbf72zrboundxgnimjtptfw0ttjun5cvloek30os7i2lzo1kcwfn8zb3rezf:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 242}, "eval/total_success_rate": 0.9428571462631226, "eval/total_success_rate_lb_0.01": 0.7908204197883606, "eval/total_success_rate_ub_0.01": 0.9943205118179321, "eval/total_success_rate_lb_0.025": 0.8324342370033264, "eval/total_success_rate_ub_0.025": 0.9927833676338196, "eval/total_success_rate_lb_0.05": 0.8533441424369812, "eval/total_success_rate_ub_0.05": 0.9836125373840332, "eval/total_success_rate_lb_0.1": 0.8608765602111816, "eval/total_success_rate_ub_0.1": 0.9836297631263733} \ No newline at end of file diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug-internal.log b/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..53dd8f9d90857cdb921d98e6c67b0f566fa6a38a --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug-internal.log @@ -0,0 +1,740 @@ +2025-01-04 20:48:47,522 INFO StreamThr :2017702 [internal.py:wandb_internal():85] W&B internal server running at pid: 2017702, started at: 2025-01-04 20:48:47.522669 +2025-01-04 20:48:47,524 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:48:47,536 INFO WriterThread:2017702 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb +2025-01-04 20:48:47,537 DEBUG SenderThread:2017702 [sender.py:send():379] send: header +2025-01-04 20:48:47,544 DEBUG SenderThread:2017702 [sender.py:send():379] send: run +2025-01-04 20:48:47,753 INFO SenderThread:2017702 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files +2025-01-04 20:48:47,753 INFO SenderThread:2017702 [sender.py:_start_run_threads():1188] run started: yxg6asez with start time 1736041727.523208 +2025-01-04 20:48:47,769 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:48:47,770 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:48:47,815 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:48:47,820 DEBUG HandlerThread:2017702 [system_info.py:__init__():26] System info init +2025-01-04 20:48:47,820 DEBUG HandlerThread:2017702 [system_info.py:__init__():41] System info init done +2025-01-04 20:48:47,820 INFO HandlerThread:2017702 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:48:47,820 INFO SystemMonitor:2017702 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:48:47,820 INFO HandlerThread:2017702 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:48:47,821 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:48:47,821 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:48:47,821 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:48:47,822 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:48:47,822 INFO SystemMonitor:2017702 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:48:47,833 DEBUG HandlerThread:2017702 [system_info.py:probe():152] Probing system +2025-01-04 20:48:47,835 DEBUG HandlerThread:2017702 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_info.py:probe():200] Probing system done +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:48:47.833575', 'startedAt': '2025-01-05T01:48:47.511040', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.498, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.986, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 623.8496360778809}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:48:47,839 INFO HandlerThread:2017702 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:48:47,839 INFO HandlerThread:2017702 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:48:47,839 DEBUG HandlerThread:2017702 [system_info.py:_save_code():44] Saving code +2025-01-04 20:48:47,845 DEBUG HandlerThread:2017702 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:48:47,845 DEBUG HandlerThread:2017702 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:48:47,907 DEBUG HandlerThread:2017702 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:48:47,907 INFO HandlerThread:2017702 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:48:47,911 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:48:47,911 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:48:47,911 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:48:47,911 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:48:48,139 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/soy2expn-wandb-metadata.json +2025-01-04 20:48:48,156 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:48:48,156 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:48:48,156 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:48:48,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:48:48,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:48,159 INFO HandlerThread:2017702 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/tb" + save: true +} +control { + relay_id: "5a5d79f676e44e83a90af2b90086f382" +} +_info { + stream_id: "yxg6asez" +} + +2025-01-04 20:48:48,159 DEBUG HandlerThread:2017702 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:48:48,160 INFO wandb-upload_1:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/irvppdhm-code/FISH/eval_robot.py +2025-01-04 20:48:48,201 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/vpqvyavf-diff.patch +2025-01-04 20:48:48,311 DEBUG SenderThread:2017702 [sender.py:send():379] send: telemetry +2025-01-04 20:48:48,312 DEBUG SenderThread:2017702 [sender.py:send():379] send: tbrecord +2025-01-04 20:48:48,312 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:48:48,312 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 with policy live +2025-01-04 20:48:48,495 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/1mydel46-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:48:48,754 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code +2025-01-04 20:48:48,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH +2025-01-04 20:48:49,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:50,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:50,754 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:51,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:51,410 DEBUG SenderThread:2017702 [sender.py:send():379] send: config +2025-01-04 20:48:52,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:52,754 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:53,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:53,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:54,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:54,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:55,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:56,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:56,755 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:57,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:58,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:58,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:58,756 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:48:59,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:00,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:00,757 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:01,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:02,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:03,156 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:03,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:03,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:03,292 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:04,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:04,758 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:05,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:06,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:06,758 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:07,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:08,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:08,759 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:09,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:09,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:10,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:11,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:12,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:13,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:14,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:14,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:15,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:16,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:17,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:18,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:18,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:18,197 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:19,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:19,228 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:19,762 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml +2025-01-04 20:49:20,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:21,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:22,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:22,763 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:23,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:24,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:25,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:25,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:26,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:27,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:28,162 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:29,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:30,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:30,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:31,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:32,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:33,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:33,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:33,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:33,807 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:49:33,807 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 with policy now +2025-01-04 20:49:33,807 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:49:33,808 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:49:33,808 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:49:33,809 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:49:34,073 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/5nvdvrdg-media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:49:34,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos +2025-01-04 20:49:34,766 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media +2025-01-04 20:49:35,070 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/ayinxzcn-media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:49:35,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:35,814 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:36,163 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:37,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:38,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:38,767 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:39,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:40,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:41,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:41,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:42,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:42,769 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:49:42,769 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:43,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:44,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:44,769 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:45,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:46,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:46,770 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:47,057 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:47,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:47,822 DEBUG SystemMonitor:2017702 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:49:47,824 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:49:48,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:49:48,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:49:48,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:48,771 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:49:49,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:50,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:51,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:52,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:52,248 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:53,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:54,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:55,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:56,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:57,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:58,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:49:58,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:49:59,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:00,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:01,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:02,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:03,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:03,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:03,205 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:03,296 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:04,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:05,168 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:06,168 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:07,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:08,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:09,161 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:09,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:10,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:10,779 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:11,170 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:12,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:13,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:13,924 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:50:13,924 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:50:13,924 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 with policy now +2025-01-04 20:50:13,924 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:50:13,925 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:50:13,925 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:50:14,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:14,184 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/95r3erdc-media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:50:14,780 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:50:14,946 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:15,074 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/oiksaks6-media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:50:15,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:16,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:16,781 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:17,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:17,824 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:50:18,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:18,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:18,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:18,781 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:19,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:20,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:20,318 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:21,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:22,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:22,783 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:23,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:23,783 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:50:24,011 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/fgzgymdz-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:50:24,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:24,548 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:50:24,548 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:50:24,548 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:50:24,784 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:50:24,784 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:25,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:25,805 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:26,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:26,784 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:27,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:28,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:28,785 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:29,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:30,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:31,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:31,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:32,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:33,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:33,157 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:33,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:34,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:35,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:36,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:36,247 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:37,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:38,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:39,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:40,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:41,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:42,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:42,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:43,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:44,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:45,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:46,175 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:47,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:47,825 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:50:47,825 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:48,157 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:50:48,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:50:48,205 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:49,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:50,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:51,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:52,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:53,164 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:53,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:54,176 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:55,179 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:55,698 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:50:55,698 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 with policy now +2025-01-04 20:50:55,698 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:50:55,699 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:50:55,699 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:50:55,700 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:50:55,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:50:55,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:50:55,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:50:55,958 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/gt7vrsj8-media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:50:56,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:56,208 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/shevj0e6-media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:50:56,794 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:57,180 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:57,795 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:50:58,169 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:50:58,179 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:50:59,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:00,182 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:00,796 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:01,181 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:02,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:03,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:03,159 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:03,183 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:03,290 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:04,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:04,797 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:05,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:05,377 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:51:05,377 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:51:05,377 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:51:05,797 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:51:05,797 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:51:06,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:06,798 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:07,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:08,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:08,747 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:09,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:10,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:10,799 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:11,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:12,185 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:13,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:14,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:14,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:15,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:16,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:17,186 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:17,825 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:51:18,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:18,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:18,205 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:19,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:19,277 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:20,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:21,187 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:22,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:23,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:24,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:25,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:25,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:26,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:27,188 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:28,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:29,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:30,167 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:30,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:31,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:32,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:33,158 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:33,158 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:33,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:34,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:35,189 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:35,275 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:36,194 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:36,807 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:37,005 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:51:37,005 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 with policy now +2025-01-04 20:51:37,006 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:51:37,007 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:51:37,007 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:51:37,007 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:51:37,190 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:37,263 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/v9uciaj8-media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:51:37,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:51:37,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:51:37,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:51:38,087 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/sq3dspj5-media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:51:38,192 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:38,808 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:39,202 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:40,199 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:40,741 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:41,196 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:42,198 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:42,810 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:43,198 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:44,201 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:44,810 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:51:45,206 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:46,214 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:46,566 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:47,209 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:47,826 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:51:48,160 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:51:48,160 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:51:48,209 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:49,209 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:50,212 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:51,210 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:52,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:52,268 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:53,216 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:54,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:55,215 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:56,215 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:57,217 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:58,173 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:51:58,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:51:59,213 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:00,214 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:01,218 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:02,216 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:03,166 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:03,166 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:03,216 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:03,288 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:04,217 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:05,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:06,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:06,448 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:06,448 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:06,449 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:06,817 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:06,818 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:07,056 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/js493f6e-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:07,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:08,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:08,818 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:09,174 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:09,220 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:10,257 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:11,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:12,219 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:12,820 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:13,220 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:14,220 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:15,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:15,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:16,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:17,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:17,826 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:52:18,165 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:18,165 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:18,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:19,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:20,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:20,303 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:20,822 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:21,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:22,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:22,823 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:23,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:24,221 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:24,824 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:25,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:26,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:26,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:27,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:28,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:29,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:30,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:31,172 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:31,222 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:32,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:33,159 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:33,159 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:33,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:34,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:34,828 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:35,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:36,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:36,881 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:37,223 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:38,235 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:38,399 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:52:38,399 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 with policy now +2025-01-04 20:52:38,402 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:52:38,402 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:38,402 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:38,403 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:38,668 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/dh8p73ib-media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:52:38,829 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:38,829 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:38,830 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:52:38,830 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval +2025-01-04 20:52:39,103 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/46c7v8ay-media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:52:39,237 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:40,238 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:40,830 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:41,242 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:42,130 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:42,233 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:43,238 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:44,236 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:44,831 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:45,235 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:46,239 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:46,832 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:47,245 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:47,827 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:52:47,827 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:48,171 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:52:48,171 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:52:48,249 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:48,354 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:52:48,354 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:52:48,578 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:52:48,833 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:48,833 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table +2025-01-04 20:52:48,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval +2025-01-04 20:52:48,839 INFO wandb-upload_2:2017702 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl940nc8n +2025-01-04 20:52:48,957 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:52:48,965 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:52:49,025 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/fvlqkexr-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:52:49,242 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:49,286 INFO SenderThread:2017702 [sender.py:send_request_log_artifact():1525] logged artifact run-yxg6asez-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMDkzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTM3NA==', 'latestArtifact': None}} +2025-01-04 20:52:49,286 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:52:49,286 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 20:52:49,286 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:49,286 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,287 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,287 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:49,287 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,287 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,288 DEBUG SenderThread:2017702 [sender.py:send():379] send: telemetry +2025-01-04 20:52:49,288 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,288 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,288 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,288 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,290 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,290 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,290 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,291 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,291 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,291 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,291 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,292 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,292 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,292 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,292 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,293 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,293 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,293 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,293 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,293 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,294 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:49,294 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:49,294 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:52:49,525 INFO wandb-upload_1:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/9tq3l4yd-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:52:49,675 INFO wandb-upload_0:2017702 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp24hn42wx +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:52:49,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval +2025-01-04 20:52:50,047 INFO SenderThread:2017702 [sender.py:send_request_log_artifact():1525] logged artifact run-yxg6asez-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMDk1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTM3Ng==', 'latestArtifact': None}} +2025-01-04 20:52:50,047 DEBUG SenderThread:2017702 [sender.py:send():379] send: files +2025-01-04 20:52:50,047 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json with policy now +2025-01-04 20:52:50,048 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:50,048 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:50,048 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:50,220 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/8ngkshr1-media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:52:50,242 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:52:50,297 DEBUG SenderThread:2017702 [sender.py:send():379] send: exit +2025-01-04 20:52:50,297 INFO SenderThread:2017702 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:send_exit():588] handling runtime: 242 +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:send_exit():594] send defer +2025-01-04 20:52:50,298 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,298 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:52:50,298 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,298 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:52:50,299 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:52:50,299 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,299 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:52:50,299 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,299 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:52:50,299 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:52:50,299 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,299 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:52:50,299 INFO HandlerThread:2017702 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:52:50,300 DEBUG SystemMonitor:2017702 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:52:50,300 DEBUG SystemMonitor:2017702 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:52:50,301 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:52:50,301 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:52:50,353 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:52:50,353 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:52:50,353 INFO HandlerThread:2017702 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:52:50,353 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,353 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:52:50,353 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:52:50,353 DEBUG SenderThread:2017702 [sender.py:send():379] send: stats +2025-01-04 20:52:50,354 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,354 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:52:50,354 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:50,354 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:52:50,354 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:52:50,354 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:50,354 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:52:50,459 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/0msw9jif-media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:52:50,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:50,834 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:52,835 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:52:56,511 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:52:56,511 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,512 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:56,512 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:52:56,512 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:52:56,512 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:52:56,513 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:52:56,513 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:56,513 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:52:56,514 DEBUG SenderThread:2017702 [sender.py:send():379] send: history +2025-01-04 20:52:56,514 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:52:56,515 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:56,515 DEBUG SenderThread:2017702 [sender.py:send():379] send: summary +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:52:56,519 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:52:56,519 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:56,519 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:52:56,519 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,519 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:52:56,523 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:52:56,667 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:52:56,668 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:56,668 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:52:56,668 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:56,668 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:52:56,837 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:52:56,837 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml +2025-01-04 20:52:57,299 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:52:59,723 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:52:59,723 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:52:59,724 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:59,724 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:52:59,724 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:52:59,724 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:52:59,724 INFO SenderThread:2017702 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:52:59,726 INFO SenderThread:2017702 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:52:59,728 INFO SenderThread:2017702 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:52:59,736 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:52:59,737 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:52:59,737 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:52:59,737 DEBUG SenderThread:2017702 [sender.py:send():379] send: artifact +2025-01-04 20:52:59,838 INFO Thread-12 :2017702 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:53:00,192 INFO wandb-upload_0:2017702 [upload_job.py:push():85] Skipped uploading /tmp/tmp24i4tet4/wandb-job.json +2025-01-04 20:53:00,192 INFO wandb-upload_1:2017702 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphf6br093 +2025-01-04 20:53:00,299 INFO wandb-upload_2:2017702 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppgbbl2ol +2025-01-04 20:53:00,302 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:00,885 INFO SenderThread:2017702 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'versionIndex': 417}}} +2025-01-04 20:53:00,885 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:00,885 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:53:00,885 INFO SenderThread:2017702 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:53:01,839 INFO SenderThread:2017702 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files +2025-01-04 20:53:01,839 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml config.yaml +2025-01-04 20:53:01,839 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json wandb-summary.json +2025-01-04 20:53:01,840 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt requirements.txt +2025-01-04 20:53:01,840 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:53:01,840 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:53:01,844 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log output.log +2025-01-04 20:53:01,844 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/diff.patch diff.patch +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 media/videos/eval/4_eval_4_11f660567809dcf580f6.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 media/videos/eval/2_eval_2_986a953e86ee8af4afd2.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 media/videos/eval/3_eval_3_e4948f3833cf9d19271a.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 media/videos/eval/0_eval_0_a83e67f40a4f92c94379.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 media/videos/eval/1_eval_1_357b7ef1e9cc666b9c49.mp4 +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json media/table/eval/total_success_rate_ci_6_57d6a400313d85e9473a.table.json +2025-01-04 20:53:01,845 INFO SenderThread:2017702 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:53:01,846 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:53:01,846 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:01,847 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:01,848 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:53:01,849 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:01,849 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:53:01,849 INFO SenderThread:2017702 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:53:02,015 INFO wandb-upload_1:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/wandb-summary.json +2025-01-04 20:53:02,020 INFO wandb-upload_2:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/requirements.txt +2025-01-04 20:53:02,035 INFO wandb-upload_0:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/config.yaml +2025-01-04 20:53:02,063 INFO wandb-upload_4:2017702 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/files/output.log +2025-01-04 20:53:02,116 INFO wandb-upload_3:2017702 [upload_job.py:push():130] Uploaded file /tmp/tmpdlzr4740wandb/edn2qdfq-events.out.tfevents.1736041728.leonmkim-ROG-Strix-G15CS-G15CS.2017514.0 +2025-01-04 20:53:02,302 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:02,302 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:02,316 INFO Thread-11 (_thread_body):2017702 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:53:02,316 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,316 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:53:02,317 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,317 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:53:02,317 INFO SenderThread:2017702 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:53:02,318 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:53:02,318 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,318 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:53:02,318 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,318 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:53:02,318 INFO SenderThread:2017702 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:53:02,487 INFO SenderThread:2017702 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:53:02,487 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:53:02,488 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,488 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,488 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:53:02,488 INFO SenderThread:2017702 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:53:02,488 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:53:02,488 INFO HandlerThread:2017702 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send():379] send: final +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send():379] send: footer +2025-01-04 20:53:02,488 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: defer +2025-01-04 20:53:02,489 INFO SenderThread:2017702 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:53:02,489 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:02,489 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:02,489 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:53:02,490 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:53:02,490 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:53:02,490 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:53:02,491 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:02,491 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:53:02,551 DEBUG SenderThread:2017702 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:53:02,551 INFO MainThread:2017702 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:53:02,552 INFO MainThread:2017702 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:53:02,552 INFO MainThread:2017702 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:53:02,552 DEBUG HandlerThread:2017702 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:53:02,552 INFO HandlerThread:2017702 [handler.py:finish():882] shutting down handler +2025-01-04 20:53:03,490 INFO WriterThread:2017702 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb +2025-01-04 20:53:03,551 INFO SenderThread:2017702 [sender.py:finish():1615] shutting down sender +2025-01-04 20:53:03,551 INFO SenderThread:2017702 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:53:03,552 INFO SenderThread:2017702 [file_pusher.py:join():175] waiting for file pusher diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug.log b/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5ccfecb5f058f0d69d6ddd351bee4eb46b868864 --- /dev/null +++ b/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Configure stats pid to 2017514 +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/settings +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:48:47,517 INFO MainThread:2017514 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug.log +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/wandb/run-20250104_204847-yxg6asez/logs/debug-internal.log +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez'} +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():619] starting backend +2025-01-04 20:48:47,518 INFO MainThread:2017514 [wandb_init.py:init():623] setting up manager +2025-01-04 20:48:47,521 INFO MainThread:2017514 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:48:47,523 INFO MainThread:2017514 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:48:47,536 INFO MainThread:2017514 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:48:47,542 INFO MainThread:2017514 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:48:47,769 INFO MainThread:2017514 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:48:47,810 INFO MainThread:2017514 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:48:47,810 INFO MainThread:2017514 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:48:48,157 INFO MainThread:2017514 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:48:48,159 INFO MainThread:2017514 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:48:48,159 INFO MainThread:2017514 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/yxg6asez/tb, True +2025-01-04 20:48:51,409 INFO MainThread:2017514 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:53:03,553 WARNING MsgRouterThr:2017514 [router.py:message_loop():77] message_loop has been closed diff --git a/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb b/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c5598ded97b303b990ec3520e2d2d9013fbab9a9 Binary files /dev/null and b/yxg6asez/wandb/run-20250104_204847-yxg6asez/run-yxg6asez.wandb differ diff --git a/z9oszy8k/episode_rosbags/episode_0_2025-01-03-20-05-17.bag b/z9oszy8k/episode_rosbags/episode_0_2025-01-03-20-05-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..3b0acc758a01b98905edeb67aed41598da21494d --- /dev/null +++ b/z9oszy8k/episode_rosbags/episode_0_2025-01-03-20-05-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6caa2932fc594af602b6d9bdb50fc90aa5740c572c6198e86234b4f4183cbd17 +size 1418080540 diff --git a/z9oszy8k/episode_rosbags/episode_1_2025-01-03-20-06-06.bag b/z9oszy8k/episode_rosbags/episode_1_2025-01-03-20-06-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..b4904426eff4205d3a9c8dbd7104df85abaa4566 --- /dev/null +++ b/z9oszy8k/episode_rosbags/episode_1_2025-01-03-20-06-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:176ed66f1bdfc01d5b07b7d1e1596ba466e443b372d0b1b6f90d38f775d30375 +size 1413650027 diff --git a/z9oszy8k/episode_rosbags/episode_2_2025-01-03-20-06-47.bag b/z9oszy8k/episode_rosbags/episode_2_2025-01-03-20-06-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..782d26ac948b5b489b2d6b739dcce143e27d4cf7 --- /dev/null +++ b/z9oszy8k/episode_rosbags/episode_2_2025-01-03-20-06-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e1d7c8c117402456d407893b24051572e58b1465951b8d5cc4e3b3ce45d6cb4 +size 1415329212 diff --git a/z9oszy8k/episode_rosbags/episode_3_2025-01-03-20-07-37.bag b/z9oszy8k/episode_rosbags/episode_3_2025-01-03-20-07-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..b715ab914e3dd59c34ded302027abe1e3ed429f7 --- /dev/null +++ b/z9oszy8k/episode_rosbags/episode_3_2025-01-03-20-07-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c29d6a4a17371bb616ee667033a2af43e6ed006f4c8c26bae3245b5b7393c143 +size 1411733589