diff --git a/.gitattributes b/.gitattributes index 9ff86c2603d67833687ad3557c9242edccd59d73..2e2e87bbfc6ddb4642a593100cf44fb3e74d14b4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -104,3 +104,43 @@ yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_900 yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 filter=lfs diff=lfs merge=lfs -text yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 filter=lfs diff=lfs merge=lfs -text yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/episode_rosbags/episode_1_2025-01-07-17-42-15.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag filter=lfs diff=lfs merge=lfs -text +jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/episode_rosbags/episode_1_2025-01-07-17-37-48.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag filter=lfs diff=lfs merge=lfs -text +j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/episode_rosbags/episode_2_2025-01-07-17-38-32.bag filter=lfs diff=lfs merge=lfs -text +j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/episode_rosbags/episode_0_2025-01-07-17-37-08.bag filter=lfs diff=lfs merge=lfs -text +j5vura5m/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +j5vura5m/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 filter=lfs diff=lfs merge=lfs -text +3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 filter=lfs diff=lfs merge=lfs -text +3b422grv/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3b422grv/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag filter=lfs diff=lfs merge=lfs -text +j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag filter=lfs diff=lfs merge=lfs -text diff --git a/1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag b/1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..d438094787caf34f3324781220d8939ef7398f10 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f78feed4ff8a94dfba4a23e5df9eeca71ee1aa7dcbcd4eb78d08e2c8cf128e2 +size 1412860245 diff --git a/1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag b/1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..3f80cc17c40e917913d668a5412e96552918d534 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20c43652a124c993d3ed9900e484f3e426829dacb450108c1628add94f6b39b1 +size 1410434229 diff --git a/1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag b/1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..7c537e2314ddfcc1d1ee530034dee60d692714f0 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35dd3627c1fca76f3d56481b1d25717179be2c20c14f9664959103ac0d4cc9c2 +size 1406647698 diff --git a/1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag b/1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..21b4cbd983420b611ab1bba22e199d4c286737ef --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bdacdadedf09f281cc585c44af876197c24da8d49d569e6e7889c57d2eb30e7 +size 1412521715 diff --git a/1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag b/1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..5116e14a8f7d7ab5198ea80da7eef11624300fdf --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04eaee91038b1ebd4b809e1f733af0f8bb40b700421f936e4673967b45b4a64 +size 1412876833 diff --git a/1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag b/1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..f0b674a5f7a7122ee862cd6ea5d0eff8d97ee864 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7854c4749bf757dbc4a8ada76206013ed99fd944a1830538b9685a4813c5f81 +size 1411637012 diff --git a/3b422grv/.hydra/config.yaml b/3b422grv/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/3b422grv/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/3b422grv/.hydra/hydra.yaml b/3b422grv/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9630e95026e376525b2430e31df6f4c8274fff3f --- /dev/null +++ b/3b422grv/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/3b422grv/.hydra/overrides.yaml b/3b422grv/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/3b422grv/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/3b422grv/episode_rosbags/aligned_depth_to_color_K.npy b/3b422grv/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3b422grv/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3b422grv/episode_rosbags/cam_tf_world.npy b/3b422grv/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/3b422grv/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/3b422grv/episode_rosbags/color_K.npy b/3b422grv/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3b422grv/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3b422grv/episode_rosbags/depth_K.npy b/3b422grv/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/3b422grv/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/3b422grv/eval_robot.log b/3b422grv/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..48727972eba280cdbe263b4ac6dac6767ed080c1 --- /dev/null +++ b/3b422grv/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:03:07,239][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:03:07,242][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/3b422grv/eval_video/0_eval.mp4 b/3b422grv/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de8ee25fbe241eef8f9944f13b112a069f519daf --- /dev/null +++ b/3b422grv/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d381be8fc37a39636946143ce961a0e9726325f87b53c10b796bff7cecf2908b +size 1203871 diff --git a/3b422grv/eval_video/1_eval.mp4 b/3b422grv/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64bc27ffaf502bf8729e910a14c22c94754c8941 --- /dev/null +++ b/3b422grv/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ca52fa14eb631b4abc059d2e4d2836c530e82cde9bbebfb4166f6247188f98 +size 1121306 diff --git a/3b422grv/eval_video/2_eval.mp4 b/3b422grv/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/3b422grv/eval_video/2_eval.mp4 differ diff --git a/3b422grv/tb/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 b/3b422grv/tb/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 new file mode 100644 index 0000000000000000000000000000000000000000..761bd1ee673e5beef8232e347541cd78caf73d9a --- /dev/null +++ b/3b422grv/tb/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af97acafbfeefa218b28bf11b31f7969fa610d6f6080ad82ca33ee336e89901d +size 1171 diff --git a/3b422grv/wandb/debug-internal.log b/3b422grv/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..eb975f0e28e08e7ffd00ef7657410f77450310c1 --- /dev/null +++ b/3b422grv/wandb/debug-internal.log @@ -0,0 +1,545 @@ +2025-01-06 17:03:13,347 INFO StreamThr :2084424 [internal.py:wandb_internal():85] W&B internal server running at pid: 2084424, started at: 2025-01-06 17:03:13.347290 +2025-01-06 17:03:13,348 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:03:13,360 INFO WriterThread:2084424 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb +2025-01-06 17:03:13,360 DEBUG SenderThread:2084424 [sender.py:send():379] send: header +2025-01-06 17:03:13,367 DEBUG SenderThread:2084424 [sender.py:send():379] send: run +2025-01-06 17:03:13,578 INFO SenderThread:2084424 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files +2025-01-06 17:03:13,578 INFO SenderThread:2084424 [sender.py:_start_run_threads():1188] run started: 3b422grv with start time 1736200993.348662 +2025-01-06 17:03:13,599 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:03:13,599 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:03:13,703 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:03:13,713 DEBUG HandlerThread:2084424 [system_info.py:__init__():26] System info init +2025-01-06 17:03:13,714 DEBUG HandlerThread:2084424 [system_info.py:__init__():41] System info init done +2025-01-06 17:03:13,714 INFO HandlerThread:2084424 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:03:13,714 INFO SystemMonitor:2084424 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:03:13,714 INFO HandlerThread:2084424 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:03:13,715 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:03:13,716 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:03:13,716 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:03:13,717 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:03:13,717 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:03:13,727 DEBUG HandlerThread:2084424 [system_info.py:probe():152] Probing system +2025-01-06 17:03:13,728 DEBUG HandlerThread:2084424 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_info.py:probe():200] Probing system done +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:03:13.727505', 'startedAt': '2025-01-06T22:03:13.338476', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.356375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.947, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.6428985595703}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:03:13,733 INFO HandlerThread:2084424 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:03:13,733 INFO HandlerThread:2084424 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_info.py:_save_code():44] Saving code +2025-01-06 17:03:13,735 DEBUG HandlerThread:2084424 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:03:13,735 DEBUG HandlerThread:2084424 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:03:13,796 DEBUG HandlerThread:2084424 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:03:13,797 INFO HandlerThread:2084424 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:03:13,800 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:03:13,800 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:03:13,800 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:03:13,800 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:03:14,054 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/v4bqqb1x-wandb-metadata.json +2025-01-06 17:03:14,062 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:03:14,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:03:14,064 DEBUG SenderThread:2084424 [sender.py:send():379] send: telemetry +2025-01-06 17:03:14,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:14,065 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:14,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:14,066 INFO HandlerThread:2084424 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb" + save: true +} +control { + relay_id: "b0cd36502b184c79a585690d8b1a1e21" +} +_info { + stream_id: "3b422grv" +} + +2025-01-06 17:03:14,066 DEBUG HandlerThread:2084424 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:03:14,071 INFO wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/axgmigee-diff.patch +2025-01-06 17:03:14,075 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/rp2vzrhc-code/FISH/eval_robot.py +2025-01-06 17:03:14,219 DEBUG SenderThread:2084424 [sender.py:send():379] send: tbrecord +2025-01-06 17:03:14,219 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:03:14,219 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 with policy live +2025-01-06 17:03:14,395 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/dk5st5le-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code +2025-01-06 17:03:15,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:16,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:16,579 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:17,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:18,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:19,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:19,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:20,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:20,151 DEBUG SenderThread:2084424 [sender.py:send():379] send: config +2025-01-06 17:03:21,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:22,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:22,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:23,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:24,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:24,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:25,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:26,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:26,582 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:27,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:28,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:29,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:29,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:29,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:29,205 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:30,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:30,583 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:31,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:32,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:32,584 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:33,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:34,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:34,394 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:35,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:36,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:36,585 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:37,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:38,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:39,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:40,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:40,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:41,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:42,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:43,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:44,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:44,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:44,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:45,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:45,163 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:45,588 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml +2025-01-06 17:03:46,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:47,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:48,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:49,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:50,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:51,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:51,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:52,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:53,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:54,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:55,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:56,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:56,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:57,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:58,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:59,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:59,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:59,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:00,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:01,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:01,157 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:02,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:03,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:04,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:05,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:06,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:07,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:07,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:08,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:09,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:10,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:11,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:12,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:12,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:13,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:13,718 DEBUG SystemMonitor:2084424 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:04:13,718 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:04:14,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:14,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:14,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:15,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:16,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:17,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:17,197 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:18,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:19,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:20,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:20,597 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:21,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:22,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:22,598 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:22,609 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:23,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:24,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:24,598 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:25,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:26,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:27,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:28,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:28,109 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:29,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:29,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:29,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:30,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:30,602 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:31,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:32,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:32,603 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:33,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:33,686 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:34,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:35,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:36,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:37,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:38,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:39,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:39,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:40,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:41,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:42,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:43,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:43,719 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:04:44,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:44,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:44,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:44,214 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:44,606 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:45,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:46,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:47,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:48,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:49,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:50,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:50,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:51,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:51,370 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:04:51,371 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 with policy now +2025-01-06 17:04:51,371 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:04:51,372 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:04:51,372 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:04:51,372 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:04:51,590 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/bhwker01-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:04:51,607 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval +2025-01-06 17:04:51,862 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/6xlmozqq-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:04:52,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:52,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:53,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:54,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:55,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:55,825 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:56,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:56,609 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:57,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:58,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:58,610 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:59,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:59,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:59,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:00,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:01,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:01,154 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:02,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:03,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:04,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:05,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:06,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:07,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:07,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:08,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:09,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:10,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:11,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:12,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:13,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:13,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:13,719 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:05:14,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:14,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:14,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:15,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:16,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:17,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:18,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:18,214 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:19,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:20,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:21,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:22,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:23,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:24,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:24,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:25,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:26,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:27,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:28,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:29,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:29,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:29,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:29,138 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:30,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:31,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:32,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:33,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:34,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:35,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:35,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:36,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:37,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:38,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:39,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:40,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:40,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:41,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:42,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:43,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:43,624 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:05:43,720 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:05:43,864 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/02f8dbvy-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:05:44,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:44,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:44,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:44,625 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:05:45,083 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:45,166 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:46,083 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:46,626 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:05:47,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:48,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:48,626 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:05:49,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:50,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:51,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:51,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:52,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:53,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:54,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:55,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:56,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:56,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:57,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:58,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:59,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:59,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:59,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:00,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:01,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:01,139 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:02,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:03,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:04,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:05,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:06,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:07,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:07,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:08,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:09,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:10,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:11,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:12,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:12,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:13,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:13,724 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:06:14,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:06:14,065 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:06:14,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:14,634 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:14,831 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:06:14,831 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 with policy now +2025-01-06 17:06:14,834 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:06:14,834 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:06:14,834 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:06:14,835 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:15,049 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/wf7exzyr-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:15,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:15,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:15,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:15,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval +2025-01-06 17:06:15,896 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/1s63098y-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:16,090 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:16,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:17,091 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:17,850 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:18,092 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:19,090 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:20,094 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:20,636 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:21,117 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:22,092 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:22,637 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:23,033 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:23,094 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:24,097 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:24,638 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:25,093 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:26,102 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:27,095 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:28,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:28,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:29,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:06:29,084 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:06:29,125 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:29,409 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:06:29,409 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:06:29,410 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-06 17:06:29,410 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:06:29,410 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:06:29,410 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:29,411 DEBUG SenderThread:2084424 [sender.py:send():379] send: telemetry +2025-01-06 17:06:29,413 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:06:29,413 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:06:29,475 DEBUG SenderThread:2084424 [sender.py:send():379] send: exit +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:send_exit():588] handling runtime: 195 +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:send_exit():594] send defer +2025-01-06 17:06:29,476 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:06:29,476 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:06:29,476 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:06:29,476 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:06:29,476 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:06:29,476 DEBUG SystemMonitor:2084424 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:06:29,476 DEBUG SystemMonitor:2084424 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:06:29,477 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:06:29,529 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:06:29,530 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:06:29,530 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:06:29,530 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,530 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:06:29,530 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:06:29,530 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:06:29,530 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,530 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:06:29,531 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,531 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:06:29,531 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:06:29,531 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,531 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:06:29,580 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/mgks1ocu-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:29,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:29,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:29,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval +2025-01-06 17:06:29,889 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/oos8d3cl-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:30,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:36,522 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:36,522 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:36,522 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:06:36,523 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,524 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:06:36,524 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:06:36,524 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:36,524 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:36,524 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:06:36,524 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:06:36,524 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:06:36,526 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:36,526 DEBUG SenderThread:2084424 [sender.py:send():379] send: summary +2025-01-06 17:06:36,526 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:36,526 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,527 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:06:36,527 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:06:36,527 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:36,527 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:06:36,527 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,527 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:06:36,530 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:36,647 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:06:36,647 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:36,647 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:06:36,648 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,648 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:06:36,648 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml +2025-01-06 17:06:36,648 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:37,492 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:38,432 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:06:38,432 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:38,432 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:38,432 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:06:38,432 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:38,432 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:06:38,433 INFO SenderThread:2084424 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:06:38,435 INFO SenderThread:2084424 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:06:38,453 INFO SenderThread:2084424 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:06:38,467 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:06:38,467 DEBUG SenderThread:2084424 [sender.py:send():379] send: artifact +2025-01-06 17:06:38,468 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:38,468 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:06:38,507 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:38,649 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:39,231 INFO wandb-upload_2:2084424 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr7_awawm +2025-01-06 17:06:39,385 INFO wandb-upload_1:2084424 [upload_job.py:push():88] Uploaded file /tmp/tmp3u9o03__/wandb-job.json +2025-01-06 17:06:39,385 INFO wandb-upload_0:2084424 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppkyoiie5 +2025-01-06 17:06:39,948 INFO SenderThread:2084424 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTg5Nzc3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-06 17:06:39,948 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:39,948 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:06:39,948 INFO SenderThread:2084424 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:06:40,650 INFO SenderThread:2084424 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files +2025-01-06 17:06:40,650 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml config.yaml +2025-01-06 17:06:40,650 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json wandb-summary.json +2025-01-06 17:06:40,652 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt requirements.txt +2025-01-06 17:06:40,652 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:06:40,652 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:06:40,653 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log output.log +2025-01-06 17:06:40,653 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch diff.patch +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:06:40,657 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:40,657 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:40,657 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:06:40,657 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:40,657 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:06:40,657 INFO SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:06:40,830 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt +2025-01-06 17:06:40,855 INFO wandb-upload_3:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:40,866 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:40,878 INFO wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml +2025-01-06 17:06:41,078 INFO Thread-11 (_thread_body):2084424 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:06:41,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,079 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:06:41,079 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,079 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:06:41,079 INFO SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:06:41,080 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:06:41,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,080 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:06:41,080 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,080 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:06:41,080 INFO SenderThread:2084424 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:06:41,286 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,286 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:06:41,286 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:06:41,286 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,287 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:06:41,287 DEBUG SenderThread:2084424 [sender.py:send():379] send: final +2025-01-06 17:06:41,287 DEBUG SenderThread:2084424 [sender.py:send():379] send: footer +2025-01-06 17:06:41,287 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,287 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:06:41,292 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:41,292 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:41,313 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:41,321 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:06:41,326 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:06:41,329 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:06:41,329 INFO HandlerThread:2084424 [handler.py:finish():882] shutting down handler +2025-01-06 17:06:42,292 INFO WriterThread:2084424 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb +2025-01-06 17:06:42,292 INFO SenderThread:2084424 [sender.py:finish():1615] shutting down sender +2025-01-06 17:06:42,293 INFO SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:06:42,293 INFO SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher diff --git a/3b422grv/wandb/debug.log b/3b422grv/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4eb55ede047bb1e55ecce41db8f8d4c5ae950097 --- /dev/null +++ b/3b422grv/wandb/debug.log @@ -0,0 +1,37 @@ +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Configure stats pid to 2084243 +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/settings +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv'} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():619] starting backend +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():623] setting up manager +2025-01-06 17:03:13,346 INFO MainThread:2084243 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:03:13,348 INFO MainThread:2084243 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:03:13,360 INFO MainThread:2084243 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:03:13,365 INFO MainThread:2084243 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:03:13,598 INFO MainThread:2084243 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:03:13,699 INFO MainThread:2084243 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:03:13,699 INFO MainThread:2084243 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:03:14,066 INFO MainThread:2084243 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:03:14,066 INFO MainThread:2084243 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb, True +2025-01-06 17:03:20,151 INFO MainThread:2084243 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:06:29,401 INFO MainThread:2084243 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/3b422grv +2025-01-06 17:06:29,403 INFO MainThread:2084243 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-06 17:06:29,407 INFO MainThread:2084243 [wandb_run.py:_restore():2352] restore +2025-01-06 17:06:29,408 INFO MainThread:2084243 [wandb_run.py:_restore():2358] restore done +2025-01-06 17:06:42,331 INFO MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:06:42,332 INFO MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:06:42,380 INFO MainThread:2084243 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py b/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml b/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81a887b366f44353cf988ac6fbc99c828e5c77d4 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736200993 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 2 + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch b/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch @@ -0,0 +1,65 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..3e5ed57 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 b/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de8ee25fbe241eef8f9944f13b112a069f519daf --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d381be8fc37a39636946143ce961a0e9726325f87b53c10b796bff7cecf2908b +size 1203871 diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 b/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64bc27ffaf502bf8729e910a14c22c94754c8941 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ca52fa14eb631b4abc059d2e4d2836c530e82cde9bbebfb4166f6247188f98 +size 1121306 diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 b/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 differ diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log b/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..acdf639eb88aaa481bb125037445eb0759b026ef --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log @@ -0,0 +1,111 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736201000.152050]: resetting environment +[INFO] [1736201000.155322]: cleared current plan +[INFO] [1736201000.156246]: moving to home +Move to target timeout: 5.000643730163574. Trans error: 0.05895647034049034. Rot error: 0.7431179070435945 +[INFO] [1736201007.959033]: reached home +[INFO] [1736201007.960956]: reset action history +[INFO] [1736201010.677988]: environment reset +Starting episode 0 +[INFO] [1736201010.680199]: resetting environment +[INFO] [1736201010.683014]: cleared current plan +[INFO] [1736201010.684005]: moving to home +[INFO] [1736201011.686442]: reached home +[INFO] [1736201011.687768]: reset action history +[INFO] [1736201014.394347]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736201059.558569]: resetting environment +[INFO] [1736201059.564067]: cleared current plan +[INFO] [1736201059.565510]: moving to home +[INFO] [1736201060.567596]: reached home +[INFO] [1736201060.568843]: reset action history +[INFO] [1736201063.276794]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/episode_rosbags/episode_0_2025-01-06-17-04-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2084243.log +started roslaunch server http://158.130.50.37:39253/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2084787] +process[print_text-2]: started with pid [2084811] +process[rosbag_record-3]: started with pid [2084812] +started bagging! +For topic gripper_width: timestamp difference is 106144366 for nearest: 1736201068065174994 - target: 1736201067959030628 at idx 65 +For topic gripper_width: timestamp difference is 102435253 for nearest: 1736201070465177439 - target: 1736201070362742186 at idx 97 +For topic gripper_width: timestamp difference is 102624022 for nearest: 1736201070665201746 - target: 1736201070562577724 at idx 99 +For topic gripper_width: timestamp difference is 101684219 for nearest: 1736201083498508579 - target: 1736201083396824360 at idx 249 +[INFO] [1736201090.318902]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736201091.124820]: Stored episode 1. +[INFO] [1736201091.125208]: Saving video... +[INFO] [1736201091.337960]: Video saved! +Starting episode 1 +[INFO] [1736201091.372020]: resetting environment +[INFO] [1736201091.376901]: cleared current plan +[INFO] [1736201091.378017]: moving to home +[INFO] [1736201094.782643]: reached home +[INFO] [1736201094.783742]: reset action history +[INFO] [1736201097.495492]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201143.068640]: resetting environment +[INFO] [1736201143.073874]: cleared current plan +[INFO] [1736201143.074064]: moving to home +[INFO] [1736201144.075095]: reached home +[INFO] [1736201144.075463]: reset action history +[INFO] [1736201146.784399]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/episode_rosbags/episode_1_2025-01-06-17-05-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2084243.log +started roslaunch server http://158.130.50.37:36435/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2084924] +process[print_text-5]: started with pid [2084925] +process[rosbag_record-6]: started with pid [2084926] +started bagging! +[INFO] [1736201173.773907]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736201174.531079]: Stored episode 2. +[INFO] [1736201174.531306]: Saving video... +[INFO] [1736201174.794614]: Video saved! +Starting episode 2 +[INFO] [1736201174.837507]: resetting environment +[INFO] [1736201174.849520]: cleared current plan +[INFO] [1736201174.849756]: moving to home +[INFO] [1736201178.352310]: reached home +[INFO] [1736201178.354759]: reset action history +[INFO] [1736201181.073285]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1736201189.372176]: Saving video... +[INFO] [1736201189.382391]: Video saved! \ No newline at end of file diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt b/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json b/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9e217e548b361ab3cf169157121755d77aa90e98 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:03:13.727505", + "startedAt": "2025-01-06T22:03:13.338476", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3375.356375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.947, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 311.6428985595703 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json b/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a17544493d08ea62688499b48b357bd94f6f9180 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "d381be8fc37a39636946143ce961a0e9726325f87b53c10b796bff7cecf2908b", "size": 1203871, "path": "media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4"}, "global_step": 133, "_timestamp": 1736201143.0684545, "_runtime": 149.7197926044464, "_step": 3, "eval/1_eval": {"_type": "video-file", "sha256": "43ca52fa14eb631b4abc059d2e4d2836c530e82cde9bbebfb4166f6247188f98", "size": 1121306, "path": "media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 195}, "eval/num_success": 0.0, "episode": 1.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.9648889303207397, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.9450690150260925, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.8587992787361145, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.07601606845855713} \ No newline at end of file diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log b/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..eb975f0e28e08e7ffd00ef7657410f77450310c1 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log @@ -0,0 +1,545 @@ +2025-01-06 17:03:13,347 INFO StreamThr :2084424 [internal.py:wandb_internal():85] W&B internal server running at pid: 2084424, started at: 2025-01-06 17:03:13.347290 +2025-01-06 17:03:13,348 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:03:13,360 INFO WriterThread:2084424 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb +2025-01-06 17:03:13,360 DEBUG SenderThread:2084424 [sender.py:send():379] send: header +2025-01-06 17:03:13,367 DEBUG SenderThread:2084424 [sender.py:send():379] send: run +2025-01-06 17:03:13,578 INFO SenderThread:2084424 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files +2025-01-06 17:03:13,578 INFO SenderThread:2084424 [sender.py:_start_run_threads():1188] run started: 3b422grv with start time 1736200993.348662 +2025-01-06 17:03:13,599 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:03:13,599 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:03:13,703 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:03:13,713 DEBUG HandlerThread:2084424 [system_info.py:__init__():26] System info init +2025-01-06 17:03:13,714 DEBUG HandlerThread:2084424 [system_info.py:__init__():41] System info init done +2025-01-06 17:03:13,714 INFO HandlerThread:2084424 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:03:13,714 INFO SystemMonitor:2084424 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:03:13,714 INFO HandlerThread:2084424 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:03:13,715 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:03:13,716 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:03:13,716 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:03:13,717 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:03:13,717 INFO SystemMonitor:2084424 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:03:13,727 DEBUG HandlerThread:2084424 [system_info.py:probe():152] Probing system +2025-01-06 17:03:13,728 DEBUG HandlerThread:2084424 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_info.py:probe():200] Probing system done +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:03:13.727505', 'startedAt': '2025-01-06T22:03:13.338476', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.356375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.947, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.6428985595703}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:03:13,733 INFO HandlerThread:2084424 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:03:13,733 INFO HandlerThread:2084424 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:03:13,733 DEBUG HandlerThread:2084424 [system_info.py:_save_code():44] Saving code +2025-01-06 17:03:13,735 DEBUG HandlerThread:2084424 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:03:13,735 DEBUG HandlerThread:2084424 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:03:13,796 DEBUG HandlerThread:2084424 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:03:13,797 INFO HandlerThread:2084424 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:03:13,800 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:03:13,800 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:03:13,800 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:03:13,800 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:03:14,054 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/v4bqqb1x-wandb-metadata.json +2025-01-06 17:03:14,062 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:03:14,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:03:14,064 DEBUG SenderThread:2084424 [sender.py:send():379] send: telemetry +2025-01-06 17:03:14,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:14,065 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:14,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:14,066 INFO HandlerThread:2084424 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb" + save: true +} +control { + relay_id: "b0cd36502b184c79a585690d8b1a1e21" +} +_info { + stream_id: "3b422grv" +} + +2025-01-06 17:03:14,066 DEBUG HandlerThread:2084424 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:03:14,071 INFO wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/axgmigee-diff.patch +2025-01-06 17:03:14,075 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/rp2vzrhc-code/FISH/eval_robot.py +2025-01-06 17:03:14,219 DEBUG SenderThread:2084424 [sender.py:send():379] send: tbrecord +2025-01-06 17:03:14,219 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:03:14,219 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 with policy live +2025-01-06 17:03:14,395 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/dk5st5le-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:03:14,580 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH +2025-01-06 17:03:14,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code +2025-01-06 17:03:15,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:16,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:16,579 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:17,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:18,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:19,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:19,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:20,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:20,151 DEBUG SenderThread:2084424 [sender.py:send():379] send: config +2025-01-06 17:03:21,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:22,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:22,581 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:23,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:24,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:24,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:25,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:26,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:26,582 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:27,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:28,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:29,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:29,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:29,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:29,205 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:30,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:30,583 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:31,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:32,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:32,584 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:33,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:34,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:34,394 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:35,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:36,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:36,585 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:03:37,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:38,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:39,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:40,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:40,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:41,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:42,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:43,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:44,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:44,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:44,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:45,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:45,163 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:45,588 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml +2025-01-06 17:03:46,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:47,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:48,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:49,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:50,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:51,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:51,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:52,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:53,068 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:54,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:55,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:56,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:03:56,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:57,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:58,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:03:59,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:03:59,063 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:03:59,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:00,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:01,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:01,157 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:02,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:03,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:04,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:05,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:06,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:07,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:07,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:08,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:09,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:10,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:11,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:12,065 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:12,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:13,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:13,718 DEBUG SystemMonitor:2084424 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:04:13,718 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:04:14,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:14,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:14,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:15,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:16,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:17,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:17,197 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:18,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:19,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:20,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:20,597 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:21,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:22,072 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:22,598 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:22,609 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:23,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:24,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:24,598 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:25,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:26,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:27,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:28,066 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:28,109 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:29,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:29,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:29,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:30,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:30,602 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:31,073 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:32,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:32,603 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:33,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:33,686 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:34,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:35,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:36,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:37,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:38,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:39,067 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:39,074 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:40,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:41,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:42,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:43,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:43,719 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:04:44,063 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:44,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:44,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:44,214 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:44,606 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:45,075 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:46,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:47,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:48,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:49,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:50,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:50,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:51,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:51,370 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:04:51,371 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 with policy now +2025-01-06 17:04:51,371 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:04:51,372 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:04:51,372 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:04:51,372 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:04:51,590 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/bhwker01-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:04:51,607 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media +2025-01-06 17:04:51,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval +2025-01-06 17:04:51,862 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/6xlmozqq-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:04:52,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:52,608 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:53,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:54,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:55,076 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:55,825 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:04:56,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:56,609 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:57,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:58,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:04:58,610 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:04:59,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:04:59,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:04:59,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:00,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:01,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:01,154 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:02,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:03,077 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:04,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:05,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:06,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:07,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:07,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:08,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:09,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:10,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:11,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:12,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:13,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:13,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:13,719 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:05:14,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:14,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:14,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:15,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:16,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:17,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:18,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:18,214 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:19,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:20,079 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:21,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:22,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:23,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:24,069 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:24,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:25,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:26,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:27,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:28,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:29,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:29,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:29,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:29,138 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:30,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:31,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:32,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:33,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:34,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:35,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:35,081 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:36,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:37,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:38,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:39,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:40,070 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:40,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:41,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:42,082 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:43,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:43,624 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:05:43,720 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:05:43,864 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/02f8dbvy-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:05:44,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:44,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:44,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:44,625 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:05:45,083 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:45,166 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:46,083 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:46,626 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:05:47,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:48,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:48,626 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:05:49,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:50,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:51,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:51,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:52,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:53,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:54,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:55,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:56,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:05:56,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:57,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:58,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:05:59,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:05:59,064 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:05:59,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:00,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:01,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:01,139 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:02,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:03,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:04,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:05,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:06,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:07,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:07,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:08,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:09,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:10,086 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:11,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:12,071 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:12,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:13,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:13,724 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:06:14,064 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:06:14,065 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:06:14,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:14,634 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:14,831 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:06:14,831 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 with policy now +2025-01-06 17:06:14,834 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:06:14,834 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:06:14,834 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:06:14,835 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:15,049 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/wf7exzyr-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:15,087 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:15,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:15,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:15,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval +2025-01-06 17:06:15,896 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/1s63098y-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:16,090 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:16,635 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:17,091 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:17,850 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:18,092 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:19,090 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:20,094 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:20,636 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:21,117 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:22,092 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:22,637 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:23,033 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:23,094 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:24,097 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:24,638 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:25,093 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:26,102 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:27,095 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:28,085 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:28,105 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:29,084 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:06:29,084 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:06:29,125 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:29,409 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:06:29,409 DEBUG SenderThread:2084424 [sender.py:send():379] send: files +2025-01-06 17:06:29,410 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-06 17:06:29,410 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:06:29,410 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:06:29,410 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:29,411 DEBUG SenderThread:2084424 [sender.py:send():379] send: telemetry +2025-01-06 17:06:29,413 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:06:29,413 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:06:29,475 DEBUG SenderThread:2084424 [sender.py:send():379] send: exit +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:send_exit():588] handling runtime: 195 +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:29,475 INFO SenderThread:2084424 [sender.py:send_exit():594] send defer +2025-01-06 17:06:29,476 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:06:29,476 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:06:29,476 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:06:29,476 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:06:29,476 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:06:29,476 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:06:29,476 DEBUG SystemMonitor:2084424 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:06:29,476 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:06:29,476 DEBUG SystemMonitor:2084424 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:06:29,477 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:06:29,529 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:06:29,530 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:06:29,530 INFO HandlerThread:2084424 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:06:29,530 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,530 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:06:29,530 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:06:29,530 DEBUG SenderThread:2084424 [sender.py:send():379] send: stats +2025-01-06 17:06:29,530 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,530 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:06:29,531 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:29,531 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:06:29,531 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:06:29,531 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:29,531 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:06:29,580 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/mgks1ocu-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:29,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:29,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:29,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval +2025-01-06 17:06:29,889 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/oos8d3cl-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:30,640 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:36,522 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:36,522 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:36,522 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:06:36,523 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,524 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:06:36,524 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:06:36,524 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:36,524 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:36,524 DEBUG SenderThread:2084424 [sender.py:send():379] send: history +2025-01-06 17:06:36,524 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:06:36,524 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:06:36,526 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:36,526 DEBUG SenderThread:2084424 [sender.py:send():379] send: summary +2025-01-06 17:06:36,526 INFO SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:06:36,526 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,527 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:06:36,527 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:06:36,527 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:36,527 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:06:36,527 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,527 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:06:36,530 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:06:36,647 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:06:36,647 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:36,647 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:06:36,648 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:36,648 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:06:36,648 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml +2025-01-06 17:06:36,648 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:37,492 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:38,432 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:06:38,432 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:38,432 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:38,432 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:06:38,432 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:38,432 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:06:38,433 INFO SenderThread:2084424 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:06:38,435 INFO SenderThread:2084424 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:06:38,453 INFO SenderThread:2084424 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:06:38,467 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:06:38,467 DEBUG SenderThread:2084424 [sender.py:send():379] send: artifact +2025-01-06 17:06:38,468 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:38,468 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:06:38,507 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:38,649 INFO Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:39,231 INFO wandb-upload_2:2084424 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr7_awawm +2025-01-06 17:06:39,385 INFO wandb-upload_1:2084424 [upload_job.py:push():88] Uploaded file /tmp/tmp3u9o03__/wandb-job.json +2025-01-06 17:06:39,385 INFO wandb-upload_0:2084424 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppkyoiie5 +2025-01-06 17:06:39,948 INFO SenderThread:2084424 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTg5Nzc3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-06 17:06:39,948 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:39,948 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:06:39,948 INFO SenderThread:2084424 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:06:40,650 INFO SenderThread:2084424 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files +2025-01-06 17:06:40,650 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml config.yaml +2025-01-06 17:06:40,650 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json wandb-summary.json +2025-01-06 17:06:40,652 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt requirements.txt +2025-01-06 17:06:40,652 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:06:40,652 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 +2025-01-06 17:06:40,653 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log output.log +2025-01-06 17:06:40,653 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch diff.patch +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 +2025-01-06 17:06:40,656 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:06:40,657 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:40,657 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:40,657 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:06:40,657 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:40,657 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:06:40,657 INFO SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:06:40,830 INFO wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt +2025-01-06 17:06:40,855 INFO wandb-upload_3:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log +2025-01-06 17:06:40,866 INFO wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json +2025-01-06 17:06:40,878 INFO wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml +2025-01-06 17:06:41,078 INFO Thread-11 (_thread_body):2084424 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:06:41,078 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,079 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:06:41,079 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,079 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:06:41,079 INFO SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:06:41,080 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:06:41,080 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,080 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:06:41,080 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,080 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:06:41,080 INFO SenderThread:2084424 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:06:41,286 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,286 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:06:41,286 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:06:41,286 INFO SenderThread:2084424 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:06:41,286 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:06:41,287 INFO HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:06:41,287 DEBUG SenderThread:2084424 [sender.py:send():379] send: final +2025-01-06 17:06:41,287 DEBUG SenderThread:2084424 [sender.py:send():379] send: footer +2025-01-06 17:06:41,287 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: defer +2025-01-06 17:06:41,287 INFO SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:06:41,292 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:06:41,292 DEBUG SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:06:41,313 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:06:41,321 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:06:41,326 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:06:41,329 DEBUG HandlerThread:2084424 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:06:41,329 INFO HandlerThread:2084424 [handler.py:finish():882] shutting down handler +2025-01-06 17:06:42,292 INFO WriterThread:2084424 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb +2025-01-06 17:06:42,292 INFO SenderThread:2084424 [sender.py:finish():1615] shutting down sender +2025-01-06 17:06:42,293 INFO SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:06:42,293 INFO SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log b/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4eb55ede047bb1e55ecce41db8f8d4c5ae950097 --- /dev/null +++ b/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log @@ -0,0 +1,37 @@ +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Configure stats pid to 2084243 +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:03:13,341 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/settings +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv'} +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():619] starting backend +2025-01-06 17:03:13,342 INFO MainThread:2084243 [wandb_init.py:init():623] setting up manager +2025-01-06 17:03:13,346 INFO MainThread:2084243 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:03:13,348 INFO MainThread:2084243 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:03:13,360 INFO MainThread:2084243 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:03:13,365 INFO MainThread:2084243 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:03:13,598 INFO MainThread:2084243 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:03:13,699 INFO MainThread:2084243 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:03:13,699 INFO MainThread:2084243 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:03:14,063 INFO MainThread:2084243 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:03:14,066 INFO MainThread:2084243 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:03:14,066 INFO MainThread:2084243 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb, True +2025-01-06 17:03:20,151 INFO MainThread:2084243 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:06:29,401 INFO MainThread:2084243 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/3b422grv +2025-01-06 17:06:29,403 INFO MainThread:2084243 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-06 17:06:29,407 INFO MainThread:2084243 [wandb_run.py:_restore():2352] restore +2025-01-06 17:06:29,408 INFO MainThread:2084243 [wandb_run.py:_restore():2358] restore done +2025-01-06 17:06:42,331 INFO MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:06:42,332 INFO MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:06:42,380 INFO MainThread:2084243 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb b/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9799ff8c1918c140bec7fdfa44a38b02b774460c Binary files /dev/null and b/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb differ diff --git a/b3d2fbi3/episode_rosbags/aligned_depth_to_color_K.npy b/b3d2fbi3/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/b3d2fbi3/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/b3d2fbi3/episode_rosbags/cam_tf_world.npy b/b3d2fbi3/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/b3d2fbi3/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/b3d2fbi3/episode_rosbags/depth_K.npy b/b3d2fbi3/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/b3d2fbi3/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/b3d2fbi3/eval_robot.log b/b3d2fbi3/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..18270e6378a0f31e34f014c2e674fc2e437d4bc6 --- /dev/null +++ b/b3d2fbi3/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 21:43:26,892][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 21:43:26,895][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/df8ok4xt/.hydra/config.yaml b/df8ok4xt/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/df8ok4xt/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/df8ok4xt/.hydra/hydra.yaml b/df8ok4xt/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..853e744a231b5dc48ff3d4ea360249a060674cae --- /dev/null +++ b/df8ok4xt/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/df8ok4xt + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/df8ok4xt/.hydra/overrides.yaml b/df8ok4xt/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/df8ok4xt/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/df8ok4xt/eval_policy.log b/df8ok4xt/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..86bf2f10086a9ca2beaee903f16cad26a836ba80 --- /dev/null +++ b/df8ok4xt/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 17:03:03,813][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:03:03,816][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 17:03:06,613][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/j5vura5m/.hydra/config.yaml b/j5vura5m/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/j5vura5m/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/j5vura5m/.hydra/hydra.yaml b/j5vura5m/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97fc28a315c82823bc90ae32281f969b0260b9a3 --- /dev/null +++ b/j5vura5m/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/j5vura5m/.hydra/overrides.yaml b/j5vura5m/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/j5vura5m/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/j5vura5m/episode_rosbags/aligned_depth_to_color_K.npy b/j5vura5m/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/j5vura5m/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/j5vura5m/episode_rosbags/cam_tf_world.npy b/j5vura5m/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/j5vura5m/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/j5vura5m/episode_rosbags/color_K.npy b/j5vura5m/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/j5vura5m/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/j5vura5m/episode_rosbags/depth_K.npy b/j5vura5m/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/j5vura5m/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag b/j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..c4356e4fa409d34b67909e3cd34b7bf91f026504 --- /dev/null +++ b/j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:819641a8054b5398caffea3774b3f9e461816c9bf29a77c9ec32e30b6b64404a +size 1409533772 diff --git a/j5vura5m/eval_robot.log b/j5vura5m/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..d48a49af3805f54ac84867801209098165f247c0 --- /dev/null +++ b/j5vura5m/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:18:28,069][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:18:28,072][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/j5vura5m/eval_video/0_eval.mp4 b/j5vura5m/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d98936bec261f7c433a3f96dc964103beaf6514 --- /dev/null +++ b/j5vura5m/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51cd14c47da033401fce5aa1e119591bc62498e0e469c63dc931c06df8db4dc7 +size 1192863 diff --git a/j5vura5m/eval_video/1_eval.mp4 b/j5vura5m/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e00169209150aef97130ada5d0343edad9835927 --- /dev/null +++ b/j5vura5m/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7111b6d98d7ffa6a81ae0d5eb00900bd97b4d17bbc2eadec084cd45420ba7256 +size 1195046 diff --git a/j5vura5m/eval_video/2_eval.mp4 b/j5vura5m/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7441272cc96b499b2e63499f262cd9141b973459 --- /dev/null +++ b/j5vura5m/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f397337911c3cdb0417f0dc598a83d75ee1e8b1ae2276f87f58308a2408510b +size 1167483 diff --git a/j5vura5m/eval_video/3_eval.mp4 b/j5vura5m/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..573f4d438dd847937246b87e7d25420d5dcb95bc --- /dev/null +++ b/j5vura5m/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d1471127c440b7c93ba7678a793d35cfbdf844064f6567a5cf421e519c9b80 +size 1179235 diff --git a/j5vura5m/eval_video/4_eval.mp4 b/j5vura5m/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef3e24dab3191d7803052a5cc5cbfb297a80bd6b --- /dev/null +++ b/j5vura5m/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdfd37d3b1b9058b2a842574c23fe729abda7c0e0b000ace39f60058f6335fca +size 1164449 diff --git a/j5vura5m/tb/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 b/j5vura5m/tb/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 new file mode 100644 index 0000000000000000000000000000000000000000..bce6c1f3e6f100347e832bb3db439597ba81758b --- /dev/null +++ b/j5vura5m/tb/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb9982bfe97bd3366e8a4191e441826084f4fae1decd13bb231c8f3159bfb0e6 +size 6539 diff --git a/j5vura5m/wandb/debug-internal.log b/j5vura5m/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6f6bac591e947597fe5e2748aae3ae482397d605 --- /dev/null +++ b/j5vura5m/wandb/debug-internal.log @@ -0,0 +1,726 @@ +2025-01-07 17:18:33,828 INFO StreamThr :2197669 [internal.py:wandb_internal():85] W&B internal server running at pid: 2197669, started at: 2025-01-07 17:18:33.828155 +2025-01-07 17:18:33,829 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:18:33,841 INFO WriterThread:2197669 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb +2025-01-07 17:18:33,841 DEBUG SenderThread:2197669 [sender.py:send():379] send: header +2025-01-07 17:18:33,849 DEBUG SenderThread:2197669 [sender.py:send():379] send: run +2025-01-07 17:18:34,177 INFO SenderThread:2197669 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files +2025-01-07 17:18:34,177 INFO SenderThread:2197669 [sender.py:_start_run_threads():1188] run started: j5vura5m with start time 1736288313.828379 +2025-01-07 17:18:34,194 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:18:34,194 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:18:34,280 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:18:34,285 DEBUG HandlerThread:2197669 [system_info.py:__init__():26] System info init +2025-01-07 17:18:34,285 DEBUG HandlerThread:2197669 [system_info.py:__init__():41] System info init done +2025-01-07 17:18:34,285 INFO HandlerThread:2197669 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:18:34,286 INFO SystemMonitor:2197669 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:18:34,286 INFO HandlerThread:2197669 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:18:34,287 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:18:34,287 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:18:34,288 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:18:34,288 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:18:34,288 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:18:34,299 DEBUG HandlerThread:2197669 [system_info.py:probe():152] Probing system +2025-01-07 17:18:34,301 DEBUG HandlerThread:2197669 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_info.py:probe():200] Probing system done +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:18:34.299929', 'startedAt': '2025-01-07T22:18:33.815607', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3931.1381249999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.913, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 750.1704368591309}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:18:34,306 INFO HandlerThread:2197669 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:18:34,306 INFO HandlerThread:2197669 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_info.py:_save_code():44] Saving code +2025-01-07 17:18:34,308 DEBUG HandlerThread:2197669 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:18:34,308 DEBUG HandlerThread:2197669 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:18:34,373 DEBUG HandlerThread:2197669 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:18:34,373 INFO HandlerThread:2197669 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:18:34,376 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:18:34,376 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:18:34,377 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:18:34,377 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:18:34,600 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/yizy7hwn-wandb-metadata.json +2025-01-07 17:18:34,621 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/6xjwu3n1-diff.patch +2025-01-07 17:18:34,627 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:18:34,627 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:18:34,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:18:34,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:18:34,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:34,632 INFO HandlerThread:2197669 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/tb" + save: true +} +control { + relay_id: "44dc74142bcb421e9e27b86ffac17c56" +} +_info { + stream_id: "j5vura5m" +} + +2025-01-07 17:18:34,632 DEBUG HandlerThread:2197669 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:18:34,656 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/t5zwytgs-code/FISH/eval_robot.py +2025-01-07 17:18:34,724 DEBUG SenderThread:2197669 [sender.py:send():379] send: telemetry +2025-01-07 17:18:34,724 DEBUG SenderThread:2197669 [sender.py:send():379] send: tbrecord +2025-01-07 17:18:34,725 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:18:34,725 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 with policy live +2025-01-07 17:18:34,924 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/scg1imby-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:18:35,178 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code +2025-01-07 17:18:35,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:36,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:37,178 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:37,489 DEBUG SenderThread:2197669 [sender.py:send():379] send: config +2025-01-07 17:18:37,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:38,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:39,178 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:39,541 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:39,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:40,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:41,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:41,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:42,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:43,180 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:43,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:44,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:44,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:45,180 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:45,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:46,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:47,181 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:47,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:48,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:49,627 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:18:49,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:18:49,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:49,765 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:50,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:51,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:52,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:53,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:54,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:55,183 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:55,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:55,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:56,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:57,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:58,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:59,184 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:59,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:00,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:00,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:01,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:02,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:03,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:04,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:04,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:04,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:05,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:05,750 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:06,186 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml +2025-01-07 17:19:06,633 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:07,633 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:08,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:09,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:10,641 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:11,507 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:11,641 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:12,642 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:13,188 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:13,642 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:14,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:15,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:16,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:16,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:17,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:18,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:19,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:19,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:19,673 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:20,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:21,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:21,775 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:22,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:23,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:24,247 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:19:24,247 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 with policy now +2025-01-07 17:19:24,248 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:19:24,249 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:19:24,249 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:19:24,250 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:19:24,507 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/2sg6ns28-media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:19:24,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:25,191 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:25,191 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos +2025-01-07 17:19:25,495 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/9wz3tx1x-media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:19:25,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:26,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:27,467 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:27,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:28,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:29,193 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:29,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:30,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:31,193 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:31,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:32,194 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:19:32,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:33,011 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:33,194 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:33,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:34,289 DEBUG SystemMonitor:2197669 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:19:34,289 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:19:34,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:34,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:34,673 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:35,195 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:35,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:36,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:37,196 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:37,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:38,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:38,962 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:39,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:40,651 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:41,651 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:42,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:43,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:44,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:44,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:45,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:46,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:47,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:48,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:49,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:49,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:49,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:49,695 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:50,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:51,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:52,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:53,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:54,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:55,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:55,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:56,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:57,657 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:58,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:59,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:00,636 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:00,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:01,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:02,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:03,659 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:03,965 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:20:03,966 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:20:03,966 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 with policy now +2025-01-07 17:20:03,966 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:20:03,966 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:20:03,967 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:20:04,204 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:20:04,204 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:20:04,205 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:20:04,220 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/s6zlm0nh-media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:20:04,290 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:20:04,485 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/mp3prmps-media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:20:04,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:04,634 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:04,660 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:05,205 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:05,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:05,757 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:06,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:07,670 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:08,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:09,206 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:09,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:10,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:10,997 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:11,207 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:11,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:12,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:13,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:14,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:15,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:16,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:16,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:17,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:18,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:19,048 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:20:19,048 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:20:19,049 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:20:19,209 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:20:19,209 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:20:19,479 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/kbkgnhnj-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:20:19,645 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:19,645 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:19,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:20,668 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:21,210 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:21,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:21,742 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:22,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:23,211 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:23,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:24,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:25,211 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:25,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:26,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:27,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:27,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:28,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:29,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:30,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:31,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:32,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:33,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:33,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:34,290 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:20:34,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:34,629 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:34,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:35,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:36,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:37,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:38,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:38,769 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:39,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:40,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:41,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:42,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:43,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:44,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:44,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:45,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:46,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:47,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:48,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:49,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:49,631 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:49,673 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:49,681 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:50,682 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:50,750 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:20:50,750 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 with policy now +2025-01-07 17:20:50,754 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:20:50,755 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:20:50,755 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:20:50,756 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:20:51,004 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/5hjha4uk-media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:20:51,482 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/4sctevlv-media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:20:51,677 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:52,676 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:53,221 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:53,678 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:54,679 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:55,221 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:55,641 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:55,679 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:56,679 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:57,681 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:58,683 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:59,223 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:59,680 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:00,642 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:00,681 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:01,684 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:02,685 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:03,684 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:04,290 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:21:04,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:04,634 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:04,683 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:04,883 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:21:04,883 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:21:04,883 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:21:05,225 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:05,225 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:21:05,225 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:21:05,693 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:06,325 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:06,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:07,226 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:07,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:08,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:09,226 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:09,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:10,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:11,639 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:11,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:12,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:13,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:14,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:15,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:16,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:16,853 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:17,230 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:17,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:18,693 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:19,230 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:19,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:19,630 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:19,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:20,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:21,231 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:21,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:22,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:22,718 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:23,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:24,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:25,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:26,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:27,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:28,638 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:28,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:29,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:30,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:31,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:32,693 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:33,639 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:33,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:34,291 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:21:34,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:34,630 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:34,691 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:35,696 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:36,116 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:21:36,116 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 with policy now +2025-01-07 17:21:36,118 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:21:36,119 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:21:36,119 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:21:36,119 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:21:36,236 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:21:36,237 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:21:36,237 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:21:36,402 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/jt8qjqd5-media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:21:36,666 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/dcuc9yfp-media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:21:36,698 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:37,237 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:37,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:38,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:39,147 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:39,698 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:40,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:41,238 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:41,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:42,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:43,239 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:43,701 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:44,189 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:44,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:45,240 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:21:45,485 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/kf70hiqt-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:21:45,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:45,931 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:21:45,932 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:21:45,932 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:21:46,241 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:21:46,710 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:47,241 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:47,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:48,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:49,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:49,631 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:49,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:49,756 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:50,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:51,243 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:51,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:52,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:53,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:54,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:55,245 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:55,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:56,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:57,245 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:57,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:58,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:59,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:00,640 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:00,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:01,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:02,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:03,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:04,291 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:22:04,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:22:04,630 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:22:04,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:05,701 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:05,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:06,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:07,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:08,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:09,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:10,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:11,640 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:11,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:12,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:13,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:14,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:15,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:16,712 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:16,935 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:16,981 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:22:16,982 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:22:16,982 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 with policy now +2025-01-07 17:22:16,982 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:16,983 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:16,983 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:17,239 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/mvfiwc3p-media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:17,252 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:17,252 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:17,252 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:17,253 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:22:17,517 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/kwq2ylqj-media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:17,715 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:18,712 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:19,253 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:19,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:22:19,649 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:22:19,719 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:20,722 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:21,254 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:21,716 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:22,564 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:22,719 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:23,727 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:24,724 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:25,255 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:25,726 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:26,728 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:26,982 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:26,982 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:26,983 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:27,256 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:27,256 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:27,721 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:27,742 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:22:27,742 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:22:27,981 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:22:28,185 INFO wandb-upload_1:2197669 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmposmbmdr9 +2025-01-07 17:22:28,258 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:28,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media +2025-01-07 17:22:28,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval +2025-01-07 17:22:28,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table +2025-01-07 17:22:28,398 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:22:28,406 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:22:28,715 INFO SenderThread:2197669 [sender.py:send_request_log_artifact():1525] logged artifact run-j5vura5m-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzU4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwMDU3NQ==', 'latestArtifact': None}} +2025-01-07 17:22:28,715 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:22:28,715 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 17:22:28,715 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,716 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send():379] send: telemetry +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,721 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,721 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,721 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,721 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,722 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:28,722 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,722 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,723 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,723 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,723 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,723 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,723 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,724 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,724 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,724 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,724 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,725 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,725 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,725 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,725 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,725 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,726 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,726 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:22:28,893 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/rm76jvqz-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:29,070 INFO wandb-upload_1:2197669 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj1keftjf +2025-01-07 17:22:29,134 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/o2yqm0ir-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval +2025-01-07 17:22:29,438 INFO SenderThread:2197669 [sender.py:send_request_log_artifact():1525] logged artifact run-j5vura5m-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzYyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwMDU4MA==', 'latestArtifact': None}} +2025-01-07 17:22:29,438 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:22:29,438 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json with policy now +2025-01-07 17:22:29,438 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:29,439 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:29,439 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:29,628 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/29vhry7j-media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:29,722 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:29,801 DEBUG SenderThread:2197669 [sender.py:send():379] send: exit +2025-01-07 17:22:29,801 INFO SenderThread:2197669 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:send_exit():588] handling runtime: 235 +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:send_exit():594] send defer +2025-01-07 17:22:29,802 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,802 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:22:29,802 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:22:29,803 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:22:29,803 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,803 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:22:29,803 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:22:29,803 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:22:29,804 DEBUG SystemMonitor:2197669 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:22:29,804 DEBUG SystemMonitor:2197669 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:22:29,854 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:22:29,854 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:22:29,854 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:22:29,855 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:22:29,855 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:22:29,855 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,855 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:22:29,855 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:22:29,855 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,855 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:22:29,894 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/m5htprle-media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:30,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:31,260 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:35,986 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:35,986 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:35,986 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:22:35,987 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:35,988 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:22:35,988 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:22:35,988 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:35,988 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:35,988 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:35,988 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:35,988 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:22:35,989 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:35,989 DEBUG SenderThread:2197669 [sender.py:send():379] send: summary +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:35,993 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:22:35,993 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:35,993 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:22:35,993 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:22:35,997 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:36,113 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:22:36,114 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:36,114 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:22:36,114 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:36,114 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:22:36,262 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml +2025-01-07 17:22:36,262 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:36,802 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:38,717 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:22:38,717 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:38,717 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:38,717 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:22:38,718 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:38,718 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:22:38,718 INFO SenderThread:2197669 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:22:38,720 INFO SenderThread:2197669 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:22:38,722 INFO SenderThread:2197669 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:22:38,736 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:22:38,736 DEBUG SenderThread:2197669 [sender.py:send():379] send: artifact +2025-01-07 17:22:38,739 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:38,739 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:22:38,803 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:39,122 INFO wandb-upload_1:2197669 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpok9kz3k0 +2025-01-07 17:22:39,124 INFO wandb-upload_2:2197669 [upload_job.py:push():85] Skipped uploading /tmp/tmpcyd64yde/wandb-job.json +2025-01-07 17:22:39,222 INFO wandb-upload_0:2197669 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0vcsooke +2025-01-07 17:22:39,263 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:39,923 INFO SenderThread:2197669 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'versionIndex': 435}}} +2025-01-07 17:22:39,923 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:39,923 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:22:39,923 INFO SenderThread:2197669 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml config.yaml +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json wandb-summary.json +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt requirements.txt +2025-01-07 17:22:40,265 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:22:40,266 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:40,266 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log output.log +2025-01-07 17:22:40,267 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch diff.patch +2025-01-07 17:22:40,267 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:22:40,271 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,271 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,271 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:22:40,273 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,273 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:22:40,273 INFO SenderThread:2197669 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:22:40,442 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml +2025-01-07 17:22:40,445 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:40,451 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt +2025-01-07 17:22:40,487 INFO wandb-upload_4:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:40,570 INFO wandb-upload_3:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/1wv0c0rb-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:40,771 INFO Thread-11 (_thread_body):2197669 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:22:40,771 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,771 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:22:40,771 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,771 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:22:40,771 INFO SenderThread:2197669 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:22:40,773 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:22:40,773 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,773 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:22:40,773 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,773 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:22:40,773 INFO SenderThread:2197669 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:22:40,804 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:40,834 INFO SenderThread:2197669 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:22:40,834 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:22:40,834 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,835 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,835 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:22:40,835 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,835 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:22:40,835 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:22:40,835 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,835 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send():379] send: final +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send():379] send: footer +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,836 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:22:40,836 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,836 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:22:40,837 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:22:40,838 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:40,838 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:22:40,839 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:22:40,839 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:40,900 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,900 INFO MainThread:2197669 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:22:40,901 INFO MainThread:2197669 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:22:40,901 INFO MainThread:2197669 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:22:40,901 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:22:40,901 INFO HandlerThread:2197669 [handler.py:finish():882] shutting down handler +2025-01-07 17:22:41,838 INFO WriterThread:2197669 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb +2025-01-07 17:22:41,900 INFO SenderThread:2197669 [sender.py:finish():1615] shutting down sender +2025-01-07 17:22:41,900 INFO SenderThread:2197669 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:22:41,900 INFO SenderThread:2197669 [file_pusher.py:join():175] waiting for file pusher diff --git a/j5vura5m/wandb/debug.log b/j5vura5m/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e09210d74bdc1ee0025b48070ea101bdfacf900 --- /dev/null +++ b/j5vura5m/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Configure stats pid to 2197489 +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/settings +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug.log +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug-internal.log +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m'} +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():619] starting backend +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():623] setting up manager +2025-01-07 17:18:33,827 INFO MainThread:2197489 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:18:33,828 INFO MainThread:2197489 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:18:33,840 INFO MainThread:2197489 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:18:33,847 INFO MainThread:2197489 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:18:34,194 INFO MainThread:2197489 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:18:34,275 INFO MainThread:2197489 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:18:34,276 INFO MainThread:2197489 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:18:34,628 INFO MainThread:2197489 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:18:34,628 INFO MainThread:2197489 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:18:34,628 INFO MainThread:2197489 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:18:34,629 INFO MainThread:2197489 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:18:34,631 INFO MainThread:2197489 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:18:34,632 INFO MainThread:2197489 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/tb, True +2025-01-07 17:18:37,489 INFO MainThread:2197489 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:22:41,902 WARNING MsgRouterThr:2197489 [router.py:message_loop():77] message_loop has been closed diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5b46a96ef8e12e2a651e41fd74b1c47d642316f --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736288313 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1af31b2ff07685aba62f0764c3073e6e9867fe4d --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch @@ -0,0 +1,21 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..6e694c7 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json new file mode 100644 index 0000000000000000000000000000000000000000..83fdaada51759e3cc35fc4b2e752a4758e04d511 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 40, 0.5, 0.3181965649491566, 0.6802346109983465, 0.3525871038731715, 0.6609474867260426, 0.3754365593444088, 0.6244643330325106, 0.3925667032814316, 0.6091313212891707]]} \ No newline at end of file diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d98936bec261f7c433a3f96dc964103beaf6514 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51cd14c47da033401fce5aa1e119591bc62498e0e469c63dc931c06df8db4dc7 +size 1192863 diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e00169209150aef97130ada5d0343edad9835927 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7111b6d98d7ffa6a81ae0d5eb00900bd97b4d17bbc2eadec084cd45420ba7256 +size 1195046 diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7441272cc96b499b2e63499f262cd9141b973459 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f397337911c3cdb0417f0dc598a83d75ee1e8b1ae2276f87f58308a2408510b +size 1167483 diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..573f4d438dd847937246b87e7d25420d5dcb95bc --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d1471127c440b7c93ba7678a793d35cfbdf844064f6567a5cf421e519c9b80 +size 1179235 diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef3e24dab3191d7803052a5cc5cbfb297a80bd6b --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdfd37d3b1b9058b2a842574c23fe729abda7c0e0b000ace39f60058f6335fca +size 1164449 diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..782db62a0532823201498ccc4dd171ff638cc77a --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736288317.489850]: resetting environment +[INFO] [1736288317.495207]: cleared current plan +[INFO] [1736288317.496072]: moving to home +[INFO] [1736288318.498733]: reached home +[INFO] [1736288318.499892]: reset action history +[INFO] [1736288321.206744]: environment reset +Starting episode 0 +[INFO] [1736288321.208678]: resetting environment +[INFO] [1736288321.211586]: cleared current plan +[INFO] [1736288321.212592]: moving to home +[INFO] [1736288322.215121]: reached home +[INFO] [1736288322.216488]: reset action history +[INFO] [1736288324.924325]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736288332.544977]: resetting environment +[INFO] [1736288332.550366]: cleared current plan +[INFO] [1736288332.551689]: moving to home +[INFO] [1736288333.553522]: reached home +[INFO] [1736288333.554857]: reset action history +[INFO] [1736288336.262755]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/episode_rosbags/episode_0_2025-01-07-17-18-56.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2197489.log +started roslaunch server http://158.130.50.37:45435/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2197806] +process[print_text-2]: started with pid [2197807] +process[rosbag_record-3]: started with pid [2197808] +started bagging! +For topic gripper_width: timestamp difference is -128826881 for nearest: 1736288351278843617 - target: 1736288351407670498 at idx 209 +[INFO] [1736288363.272049]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736288364.026061]: Stored episode 1. +[INFO] [1736288364.026680]: Saving video... +[INFO] [1736288364.232178]: Video saved! +Starting episode 1 +[INFO] [1736288364.250220]: resetting environment +[INFO] [1736288364.262937]: cleared current plan +[INFO] [1736288364.263635]: moving to home +[INFO] [1736288367.466672]: reached home +[INFO] [1736288367.467528]: reset action history +[INFO] [1736288370.175445]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288372.004306]: resetting environment +[INFO] [1736288372.009308]: cleared current plan +[INFO] [1736288372.009647]: moving to home +[INFO] [1736288373.011238]: reached home +[INFO] [1736288373.011660]: reset action history +[INFO] [1736288375.717798]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/episode_rosbags/episode_1_2025-01-07-17-19-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2197489.log +started roslaunch server http://158.130.50.37:35317/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2197928] +process[print_text-5]: started with pid [2197929] +process[rosbag_record-6]: started with pid [2197930] +started bagging! +[INFO] [1736288402.671873]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736288403.447862]: Stored episode 2. +[INFO] [1736288403.448328]: Saving video... +[INFO] [1736288403.922230]: Video saved! +Starting episode 2 +[INFO] [1736288403.958773]: resetting environment +[INFO] [1736288403.967062]: cleared current plan +[INFO] [1736288403.967641]: moving to home +[INFO] [1736288407.175343]: reached home +[INFO] [1736288407.178536]: reset action history +[INFO] [1736288409.952904]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288418.589738]: resetting environment +[INFO] [1736288418.597971]: cleared current plan +[INFO] [1736288418.598705]: moving to home +[INFO] [1736288419.602467]: reached home +[INFO] [1736288419.602885]: reset action history +[INFO] [1736288422.311422]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2197489.log +started roslaunch server http://158.130.50.37:44931/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2198077] +process[print_text-8]: started with pid [2198078] +process[rosbag_record-9]: started with pid [2198079] +started bagging! +[INFO] [1736288449.260920]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736288450.082807]: Stored episode 3. +[INFO] [1736288450.085495]: Saving video... +[WARN] [1736288450.610850]: Plan exhausted +[WARN] [1736288450.650743]: Plan exhausted +[WARN] [1736288450.690792]: Plan exhausted +[WARN] [1736288450.730982]: Plan exhausted +[WARN] [1736288450.770225]: Plan exhausted +[INFO] [1736288450.707788]: Video saved! +Starting episode 3 +[INFO] [1736288450.764675]: resetting environment +[INFO] [1736288450.784140]: cleared current plan +[INFO] [1736288450.785433]: moving to home +[INFO] [1736288453.989678]: reached home +[INFO] [1736288453.996486]: reset action history +[INFO] [1736288456.774509]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288464.251394]: resetting environment +[INFO] [1736288464.276225]: cleared current plan +[INFO] [1736288464.278969]: moving to home +[INFO] [1736288465.283012]: reached home +[INFO] [1736288465.287974]: reset action history +[INFO] [1736288468.063159]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/episode_rosbags/episode_3_2025-01-07-17-21-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2197489.log +started roslaunch server http://158.130.50.37:40473/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2198225] +process[print_text-11]: started with pid [2198226] +process[rosbag_record-12]: started with pid [2198227] +started bagging! +For topic gripper_width: timestamp difference is 103288423 for nearest: 1736288475045509349 - target: 1736288474942220926 at idx 89 +For topic gripper_width: timestamp difference is 103680607 for nearest: 1736288475245508906 - target: 1736288475141828299 at idx 91 +For topic gripper_width: timestamp difference is 106956465 for nearest: 1736288476845540030 - target: 1736288476738583565 at idx 113 +For topic gripper_width: timestamp difference is 108525319 for nearest: 1736288477645510001 - target: 1736288477536984682 at idx 125 +For topic gripper_width: timestamp difference is 110527514 for nearest: 1736288478645508288 - target: 1736288478534980774 at idx 138 +For topic gripper_width: timestamp difference is 110942765 for nearest: 1736288478845507785 - target: 1736288478734565020 at idx 140 +[INFO] [1736288495.023935]: Storing episode... +[pub_text-10] killing on exit +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1736288495.808592]: Stored episode 4. +[INFO] [1736288495.812743]: Saving video... +[INFO] [1736288496.066159]: Video saved! +Starting episode 4 +[INFO] [1736288496.123356]: resetting environment +[INFO] [1736288496.146248]: cleared current plan +[INFO] [1736288496.146503]: moving to home +[INFO] [1736288499.351189]: reached home +[INFO] [1736288499.360321]: reset action history +[INFO] [1736288502.145944]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288505.021445]: resetting environment +[INFO] [1736288505.039873]: cleared current plan +[INFO] [1736288505.041325]: moving to home +[INFO] [1736288506.045084]: reached home +[INFO] [1736288506.050663]: reset action history +[INFO] [1736288508.823242]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/episode_rosbags/episode_4_2025-01-07-17-21-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2197489.log +started roslaunch server http://158.130.50.37:41057/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2198369] +process[print_text-14]: started with pid [2198370] +process[rosbag_record-15]: started with pid [2198394] +started bagging! +For topic gripper_width: timestamp difference is 103457871 for nearest: 1736288515245542470 - target: 1736288515142084599 at idx 91 +[INFO] [1736288535.801399]: Storing episode... +[rosbag_record-15] killing on exit[pub_text-13] killing on exit +[print_text-14] killing on exit +[WARN] [1736288536.930283]: Plan exhausted +[WARN] [1736288536.971725]: Plan exhausted +[INFO] [1736288536.590758]: Stored episode 5. +[INFO] [1736288536.591602]: Saving video... +[INFO] [1736288536.940008]: Video saved! +[INFO] [1736288536.981781]: resetting environment +[INFO] [1736288536.998767]: cleared current plan +[INFO] [1736288536.999044]: moving to home +[INFO] [1736288539.601592]: reached home +[INFO] [1736288539.607643]: reset action history +[INFO] [1736288542.563529]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0ce221b648bae195f4d26550c84e33f8ea0b8f69 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:18:34.299929", + "startedAt": "2025-01-07T22:18:33.815607", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3931.1381249999995, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.913, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 750.1704368591309 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..badc218620bb046ba38beaf683f8bf8ac1cc54b1 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "51cd14c47da033401fce5aa1e119591bc62498e0e469c63dc931c06df8db4dc7", "size": 1192863, "path": "media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4"}, "global_step": 664, "_timestamp": 1736288548.30691, "_runtime": 234.47853112220764, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "7111b6d98d7ffa6a81ae0d5eb00900bd97b4d17bbc2eadec084cd45420ba7256", "size": 1195046, "path": "media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4"}, "eval/num_success": 0.0, "episode": 40.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "5f397337911c3cdb0417f0dc598a83d75ee1e8b1ae2276f87f58308a2408510b", "size": 1167483, "path": "media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "67d1471127c440b7c93ba7678a793d35cfbdf844064f6567a5cf421e519c9b80", "size": 1179235, "path": "media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "fdfd37d3b1b9058b2a842574c23fe729abda7c0e0b000ace39f60058f6335fca", "size": 1164449, "path": "media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://4keg0svs6mb0hnmzte3qljd0k8hi6q014d8rxf241960d8xxk8i8zkkqj7s23zerxkevk74tkkldc0oumpg5grroxo9q1s1qvdx7bfpzkplme3r7g56qgguwirl53piy/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://l3nh64g1rl8jzk2rorvglf0yj94usefhe38tt74lbd77lcvhqkyvy0u64ukrcay587czeq2dct3aiwjjoswe7ub2dtufi1fxusc7w84s1ttaancpua26kq6ex7u2qug6:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 40, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.3181965649491566, "total_success_rate_ub_0.01": 0.6802346109983465, "total_success_rate_lb_0.025": 0.3525871038731715, "total_success_rate_ub_0.025": 0.6609474867260426, "total_success_rate_lb_0.05": 0.3754365593444088, "total_success_rate_ub_0.05": 0.6244643330325106, "total_success_rate_lb_0.1": 0.3925667032814316, "total_success_rate_ub_0.1": 0.6091313212891707, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "cedb3ebe8ba6d94078f51091161899d4e95712db9c7c24c93bc79f23bb298c2d", "size": 505, "artifact_path": "wandb-client-artifact://unahldrbo7xjtrra0wbkw7bjirpxqafatnqdbfvz86izdlhf6k4hbiasl2u46ulmzfeeooh3769g9ov2fdk1ncwh740lb2u5uskwfpv36tweiksbd8ejafwa0cwf2fui/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://0ws2tpc1zjqqu81i90oco3jtnmveicyu8bfrbf3156g9wsrv65m4s6l91mx1qhgu04i0w3i0h3a5iizknfc6tz527h8g7vc6i4gblrufqku4vwy59qpeuvchs7rzlf4w:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 235}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.318196564912796, "eval/total_success_rate_ub_0.01": 0.6802346110343933, "eval/total_success_rate_lb_0.025": 0.35258710384368896, "eval/total_success_rate_ub_0.025": 0.6609475016593933, "eval/total_success_rate_lb_0.05": 0.37543657422065735, "eval/total_success_rate_ub_0.05": 0.6244643330574036, "eval/total_success_rate_lb_0.1": 0.3925667107105255, "eval/total_success_rate_ub_0.1": 0.6091313362121582} \ No newline at end of file diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug-internal.log b/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6f6bac591e947597fe5e2748aae3ae482397d605 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug-internal.log @@ -0,0 +1,726 @@ +2025-01-07 17:18:33,828 INFO StreamThr :2197669 [internal.py:wandb_internal():85] W&B internal server running at pid: 2197669, started at: 2025-01-07 17:18:33.828155 +2025-01-07 17:18:33,829 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:18:33,841 INFO WriterThread:2197669 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb +2025-01-07 17:18:33,841 DEBUG SenderThread:2197669 [sender.py:send():379] send: header +2025-01-07 17:18:33,849 DEBUG SenderThread:2197669 [sender.py:send():379] send: run +2025-01-07 17:18:34,177 INFO SenderThread:2197669 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files +2025-01-07 17:18:34,177 INFO SenderThread:2197669 [sender.py:_start_run_threads():1188] run started: j5vura5m with start time 1736288313.828379 +2025-01-07 17:18:34,194 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:18:34,194 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:18:34,280 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:18:34,285 DEBUG HandlerThread:2197669 [system_info.py:__init__():26] System info init +2025-01-07 17:18:34,285 DEBUG HandlerThread:2197669 [system_info.py:__init__():41] System info init done +2025-01-07 17:18:34,285 INFO HandlerThread:2197669 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:18:34,286 INFO SystemMonitor:2197669 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:18:34,286 INFO HandlerThread:2197669 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:18:34,287 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:18:34,287 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:18:34,288 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:18:34,288 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:18:34,288 INFO SystemMonitor:2197669 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:18:34,299 DEBUG HandlerThread:2197669 [system_info.py:probe():152] Probing system +2025-01-07 17:18:34,301 DEBUG HandlerThread:2197669 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_info.py:probe():200] Probing system done +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:18:34.299929', 'startedAt': '2025-01-07T22:18:33.815607', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3931.1381249999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.913, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 750.1704368591309}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:18:34,306 INFO HandlerThread:2197669 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:18:34,306 INFO HandlerThread:2197669 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:18:34,306 DEBUG HandlerThread:2197669 [system_info.py:_save_code():44] Saving code +2025-01-07 17:18:34,308 DEBUG HandlerThread:2197669 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:18:34,308 DEBUG HandlerThread:2197669 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:18:34,373 DEBUG HandlerThread:2197669 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:18:34,373 INFO HandlerThread:2197669 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:18:34,376 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:18:34,376 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:18:34,377 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:18:34,377 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:18:34,600 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/yizy7hwn-wandb-metadata.json +2025-01-07 17:18:34,621 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/6xjwu3n1-diff.patch +2025-01-07 17:18:34,627 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:18:34,627 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:18:34,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:18:34,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:18:34,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:34,632 INFO HandlerThread:2197669 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/tb" + save: true +} +control { + relay_id: "44dc74142bcb421e9e27b86ffac17c56" +} +_info { + stream_id: "j5vura5m" +} + +2025-01-07 17:18:34,632 DEBUG HandlerThread:2197669 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:18:34,656 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/t5zwytgs-code/FISH/eval_robot.py +2025-01-07 17:18:34,724 DEBUG SenderThread:2197669 [sender.py:send():379] send: telemetry +2025-01-07 17:18:34,724 DEBUG SenderThread:2197669 [sender.py:send():379] send: tbrecord +2025-01-07 17:18:34,725 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:18:34,725 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 with policy live +2025-01-07 17:18:34,924 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/scg1imby-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:18:35,178 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH +2025-01-07 17:18:35,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code +2025-01-07 17:18:35,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:36,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:37,178 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:37,489 DEBUG SenderThread:2197669 [sender.py:send():379] send: config +2025-01-07 17:18:37,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:38,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:39,178 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:39,541 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:39,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:40,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:41,179 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:41,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:42,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:43,180 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:43,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:44,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:44,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:45,180 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:45,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:46,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:47,181 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:47,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:48,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:49,627 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:18:49,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:18:49,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:49,765 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:50,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:51,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:52,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:53,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:54,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:55,183 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:55,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:18:55,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:56,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:57,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:58,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:18:59,184 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:18:59,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:00,629 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:00,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:01,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:02,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:03,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:04,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:04,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:04,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:05,632 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:05,750 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:06,186 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml +2025-01-07 17:19:06,633 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:07,633 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:08,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:09,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:10,641 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:11,507 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:11,641 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:12,642 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:13,188 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:13,642 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:14,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:15,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:16,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:16,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:17,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:18,643 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:19,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:19,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:19,673 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:20,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:21,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:21,775 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:22,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:23,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:24,247 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:19:24,247 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 with policy now +2025-01-07 17:19:24,248 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:19:24,249 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:19:24,249 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:19:24,250 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:19:24,507 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/2sg6ns28-media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:19:24,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:25,191 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:25,191 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media +2025-01-07 17:19:25,192 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos +2025-01-07 17:19:25,495 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/9wz3tx1x-media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:19:25,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:26,648 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:27,467 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:27,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:28,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:29,193 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:29,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:30,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:31,193 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:31,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:32,194 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:19:32,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:33,011 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:33,194 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:33,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:34,289 DEBUG SystemMonitor:2197669 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:19:34,289 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:19:34,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:34,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:34,673 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:35,195 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:35,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:36,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:37,196 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:19:37,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:38,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:38,962 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:39,650 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:40,651 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:41,651 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:42,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:43,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:44,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:44,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:45,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:46,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:47,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:48,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:49,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:19:49,628 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:19:49,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:49,695 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:50,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:51,655 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:52,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:53,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:54,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:55,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:19:55,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:56,656 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:57,657 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:58,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:19:59,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:00,636 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:00,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:01,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:02,658 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:03,659 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:03,965 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:20:03,966 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:20:03,966 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 with policy now +2025-01-07 17:20:03,966 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:20:03,966 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:20:03,967 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:20:04,204 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:20:04,204 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:20:04,205 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:20:04,220 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/s6zlm0nh-media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:20:04,290 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:20:04,485 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/mp3prmps-media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:20:04,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:04,634 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:04,660 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:05,205 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:05,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:05,757 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:06,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:07,670 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:08,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:09,206 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:09,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:10,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:10,997 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:11,207 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:11,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:12,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:13,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:14,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:15,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:16,631 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:16,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:17,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:18,663 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:19,048 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:20:19,048 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:20:19,049 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:20:19,209 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:20:19,209 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:20:19,479 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/kbkgnhnj-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:20:19,645 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:19,645 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:19,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:20,668 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:21,210 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:21,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:21,742 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:22,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:23,211 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:23,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:24,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:25,211 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:25,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:26,664 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:27,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:27,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:28,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:29,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:30,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:31,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:32,665 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:33,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:33,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:34,290 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:20:34,628 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:34,629 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:34,669 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:35,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:36,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:37,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:38,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:38,769 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:39,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:40,666 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:41,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:42,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:43,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:44,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:44,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:45,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:46,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:47,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:48,667 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:49,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:20:49,631 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:20:49,673 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:49,681 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:50,682 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:50,750 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:20:50,750 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 with policy now +2025-01-07 17:20:50,754 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:20:50,755 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:20:50,755 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:20:50,756 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:20:51,004 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/5hjha4uk-media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:20:51,220 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:20:51,482 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/4sctevlv-media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:20:51,677 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:52,676 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:53,221 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:53,678 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:54,679 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:55,221 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:55,641 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:20:55,679 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:56,679 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:57,681 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:58,683 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:20:59,223 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:20:59,680 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:00,642 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:00,681 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:01,684 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:02,685 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:03,684 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:04,290 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:21:04,634 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:04,634 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:04,683 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:04,883 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:21:04,883 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:21:04,883 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:21:05,225 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:05,225 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:21:05,225 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:21:05,693 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:06,325 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:06,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:07,226 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:07,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:08,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:09,226 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:09,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:10,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:11,639 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:11,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:12,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:13,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:14,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:15,686 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:16,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:16,853 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:17,230 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:17,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:18,693 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:19,230 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:19,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:19,630 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:19,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:20,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:21,231 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:21,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:22,687 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:22,718 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:23,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:24,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:25,688 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:26,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:27,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:28,638 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:28,689 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:29,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:30,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:31,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:32,693 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:33,639 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:33,690 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:34,291 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:21:34,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:34,630 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:34,691 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:35,696 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:36,116 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:21:36,116 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 with policy now +2025-01-07 17:21:36,118 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:21:36,119 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:21:36,119 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:21:36,119 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:21:36,236 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:21:36,237 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:21:36,237 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:21:36,402 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/jt8qjqd5-media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:21:36,666 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/dcuc9yfp-media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:21:36,698 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:37,237 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:37,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:38,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:39,147 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:39,698 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:40,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:41,238 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:41,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:42,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:43,239 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:43,701 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:44,189 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:44,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:45,240 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:21:45,485 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/kf70hiqt-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:21:45,702 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:45,931 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:21:45,932 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:21:45,932 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:21:46,241 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:21:46,710 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:47,241 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:47,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:48,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:49,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:21:49,631 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:21:49,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:49,756 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:50,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:51,243 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:51,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:52,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:53,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:54,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:55,245 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:21:55,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:56,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:57,245 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:21:57,704 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:58,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:21:59,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:00,640 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:00,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:01,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:02,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:03,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:04,291 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:22:04,630 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:22:04,630 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:22:04,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:05,701 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:05,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:06,705 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:07,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:08,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:09,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:10,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:11,640 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:11,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:12,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:13,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:14,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:15,706 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:16,712 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:16,935 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:16,981 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:22:16,982 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:22:16,982 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 with policy now +2025-01-07 17:22:16,982 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:16,983 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:16,983 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:17,239 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/mvfiwc3p-media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:17,252 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:17,252 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:17,252 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:17,253 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval +2025-01-07 17:22:17,517 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/kwq2ylqj-media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:17,715 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:18,712 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:19,253 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:19,649 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:22:19,649 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:22:19,719 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:20,722 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:21,254 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:21,716 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:22,564 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:22,719 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:23,727 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:24,724 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:25,255 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:25,726 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:26,728 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:26,982 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:26,982 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:26,983 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:27,256 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:27,256 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:27,721 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:27,742 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:22:27,742 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:22:27,981 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:22:28,185 INFO wandb-upload_1:2197669 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmposmbmdr9 +2025-01-07 17:22:28,258 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:28,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media +2025-01-07 17:22:28,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval +2025-01-07 17:22:28,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table +2025-01-07 17:22:28,398 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:22:28,406 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:22:28,715 INFO SenderThread:2197669 [sender.py:send_request_log_artifact():1525] logged artifact run-j5vura5m-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzU4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwMDU3NQ==', 'latestArtifact': None}} +2025-01-07 17:22:28,715 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:22:28,715 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 17:22:28,715 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,716 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send():379] send: telemetry +2025-01-07 17:22:28,716 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,721 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,721 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,721 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,721 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,722 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:28,722 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,722 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,723 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,723 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,723 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,723 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,723 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,724 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,724 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,724 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,724 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,725 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,725 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,725 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,725 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,725 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:28,726 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:28,726 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:22:28,893 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/rm76jvqz-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:29,070 INFO wandb-upload_1:2197669 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj1keftjf +2025-01-07 17:22:29,134 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/o2yqm0ir-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:29,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval +2025-01-07 17:22:29,438 INFO SenderThread:2197669 [sender.py:send_request_log_artifact():1525] logged artifact run-j5vura5m-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzYyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwMDU4MA==', 'latestArtifact': None}} +2025-01-07 17:22:29,438 DEBUG SenderThread:2197669 [sender.py:send():379] send: files +2025-01-07 17:22:29,438 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json with policy now +2025-01-07 17:22:29,438 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:29,439 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:29,439 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:29,628 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/29vhry7j-media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:29,722 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:29,801 DEBUG SenderThread:2197669 [sender.py:send():379] send: exit +2025-01-07 17:22:29,801 INFO SenderThread:2197669 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:send_exit():588] handling runtime: 235 +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:send_exit():594] send defer +2025-01-07 17:22:29,802 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,802 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:22:29,802 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:22:29,802 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:22:29,803 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:22:29,803 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,803 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:22:29,803 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:22:29,803 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:22:29,803 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:22:29,804 DEBUG SystemMonitor:2197669 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:22:29,804 DEBUG SystemMonitor:2197669 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:22:29,854 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:22:29,854 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:22:29,854 INFO HandlerThread:2197669 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:22:29,855 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:22:29,855 DEBUG SenderThread:2197669 [sender.py:send():379] send: stats +2025-01-07 17:22:29,855 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,855 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:22:29,855 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:22:29,855 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:22:29,855 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:29,855 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:22:29,894 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/m5htprle-media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:30,259 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:31,260 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:35,986 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:35,986 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:35,986 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:22:35,987 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:35,988 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:22:35,988 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:22:35,988 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:35,988 DEBUG SenderThread:2197669 [sender.py:send():379] send: history +2025-01-07 17:22:35,988 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:22:35,988 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:35,988 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:22:35,989 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:35,989 DEBUG SenderThread:2197669 [sender.py:send():379] send: summary +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:22:35,993 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:22:35,993 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:35,993 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:22:35,993 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:35,993 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:22:35,997 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:22:36,113 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:22:36,114 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:36,114 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:22:36,114 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:36,114 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:22:36,262 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml +2025-01-07 17:22:36,262 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:36,802 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:38,717 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:22:38,717 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:38,717 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:38,717 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:22:38,718 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:38,718 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:22:38,718 INFO SenderThread:2197669 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:22:38,720 INFO SenderThread:2197669 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:22:38,722 INFO SenderThread:2197669 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:22:38,736 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:22:38,736 DEBUG SenderThread:2197669 [sender.py:send():379] send: artifact +2025-01-07 17:22:38,739 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:38,739 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:22:38,803 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:39,122 INFO wandb-upload_1:2197669 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpok9kz3k0 +2025-01-07 17:22:39,124 INFO wandb-upload_2:2197669 [upload_job.py:push():85] Skipped uploading /tmp/tmpcyd64yde/wandb-job.json +2025-01-07 17:22:39,222 INFO wandb-upload_0:2197669 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0vcsooke +2025-01-07 17:22:39,263 INFO Thread-12 :2197669 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:39,923 INFO SenderThread:2197669 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'versionIndex': 435}}} +2025-01-07 17:22:39,923 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:39,923 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:22:39,923 INFO SenderThread:2197669 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml config.yaml +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json wandb-summary.json +2025-01-07 17:22:40,264 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt requirements.txt +2025-01-07 17:22:40,265 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:22:40,266 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:40,266 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log output.log +2025-01-07 17:22:40,267 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/diff.patch diff.patch +2025-01-07 17:22:40,267 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json media/table/eval/total_success_rate_ci_6_cedb3ebe8ba6d94078f5.table.json +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:22:40,270 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:22:40,271 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,271 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,271 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:22:40,273 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,273 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:22:40,273 INFO SenderThread:2197669 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:22:40,442 INFO wandb-upload_1:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/config.yaml +2025-01-07 17:22:40,445 INFO wandb-upload_0:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/wandb-summary.json +2025-01-07 17:22:40,451 INFO wandb-upload_2:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/requirements.txt +2025-01-07 17:22:40,487 INFO wandb-upload_4:2197669 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/files/output.log +2025-01-07 17:22:40,570 INFO wandb-upload_3:2197669 [upload_job.py:push():130] Uploaded file /tmp/tmpzsb1pqfewandb/1wv0c0rb-events.out.tfevents.1736288314.leonmkim-ROG-Strix-G15CS-G15CS.2197489.0 +2025-01-07 17:22:40,771 INFO Thread-11 (_thread_body):2197669 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:22:40,771 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,771 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:22:40,771 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,771 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:22:40,771 INFO SenderThread:2197669 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:22:40,773 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:22:40,773 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,773 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:22:40,773 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,773 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:22:40,773 INFO SenderThread:2197669 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:22:40,804 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:40,834 INFO SenderThread:2197669 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:22:40,834 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:22:40,834 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,835 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,835 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:22:40,835 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,835 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:22:40,835 INFO SenderThread:2197669 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:22:40,835 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:22:40,835 INFO HandlerThread:2197669 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send():379] send: final +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send():379] send: footer +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: defer +2025-01-07 17:22:40,836 INFO SenderThread:2197669 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:22:40,836 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:40,836 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,836 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:22:40,837 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:22:40,838 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:22:40,838 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:22:40,839 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:22:40,839 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:22:40,900 DEBUG SenderThread:2197669 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:22:40,900 INFO MainThread:2197669 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:22:40,901 INFO MainThread:2197669 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:22:40,901 INFO MainThread:2197669 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:22:40,901 DEBUG HandlerThread:2197669 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:22:40,901 INFO HandlerThread:2197669 [handler.py:finish():882] shutting down handler +2025-01-07 17:22:41,838 INFO WriterThread:2197669 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb +2025-01-07 17:22:41,900 INFO SenderThread:2197669 [sender.py:finish():1615] shutting down sender +2025-01-07 17:22:41,900 INFO SenderThread:2197669 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:22:41,900 INFO SenderThread:2197669 [file_pusher.py:join():175] waiting for file pusher diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug.log b/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e09210d74bdc1ee0025b48070ea101bdfacf900 --- /dev/null +++ b/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Configure stats pid to 2197489 +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/settings +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug.log +2025-01-07 17:18:33,822 INFO MainThread:2197489 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/wandb/run-20250107_171833-j5vura5m/logs/debug-internal.log +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m'} +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():619] starting backend +2025-01-07 17:18:33,823 INFO MainThread:2197489 [wandb_init.py:init():623] setting up manager +2025-01-07 17:18:33,827 INFO MainThread:2197489 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:18:33,828 INFO MainThread:2197489 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:18:33,840 INFO MainThread:2197489 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:18:33,847 INFO MainThread:2197489 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:18:34,194 INFO MainThread:2197489 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:18:34,275 INFO MainThread:2197489 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:18:34,276 INFO MainThread:2197489 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:18:34,628 INFO MainThread:2197489 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:18:34,628 INFO MainThread:2197489 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:18:34,628 INFO MainThread:2197489 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:18:34,629 INFO MainThread:2197489 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:18:34,631 INFO MainThread:2197489 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:18:34,632 INFO MainThread:2197489 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m/tb, True +2025-01-07 17:18:37,489 INFO MainThread:2197489 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:22:41,902 WARNING MsgRouterThr:2197489 [router.py:message_loop():77] message_loop has been closed diff --git a/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb b/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c33ebf85ea6a45c662208245590c4186f71d0b5c Binary files /dev/null and b/j5vura5m/wandb/run-20250107_171833-j5vura5m/run-j5vura5m.wandb differ diff --git a/jqiikotd/.hydra/config.yaml b/jqiikotd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/jqiikotd/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/jqiikotd/.hydra/hydra.yaml b/jqiikotd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4c4b60969d7bd3572a15b4031193a95db33bb86 --- /dev/null +++ b/jqiikotd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/jqiikotd/.hydra/overrides.yaml b/jqiikotd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/jqiikotd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/jqiikotd/episode_rosbags/aligned_depth_to_color_K.npy b/jqiikotd/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/jqiikotd/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/jqiikotd/episode_rosbags/cam_tf_world.npy b/jqiikotd/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/jqiikotd/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/jqiikotd/episode_rosbags/color_K.npy b/jqiikotd/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/jqiikotd/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/jqiikotd/episode_rosbags/depth_K.npy b/jqiikotd/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/jqiikotd/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/jqiikotd/eval_robot.log b/jqiikotd/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..f2d831f14b601e55698e0f8f9f674208c1868c4b --- /dev/null +++ b/jqiikotd/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:23:01,643][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:23:01,646][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/jqiikotd/eval_video/0_eval.mp4 b/jqiikotd/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8d85b4c6b9a27fc326066ba36bb62c0d381908a9 --- /dev/null +++ b/jqiikotd/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c751aa70732cb0acb7db0ecff477db74aa07b780d478d466f4d27e7581e3b7 +size 1171819 diff --git a/jqiikotd/eval_video/1_eval.mp4 b/jqiikotd/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddd11416756a8e46ba2a39342105bab79d6bc894 --- /dev/null +++ b/jqiikotd/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac128dda904a091cb30d97d31d39844ceca216ce6955f12d5233053170f48e0 +size 1178239 diff --git a/jqiikotd/eval_video/2_eval.mp4 b/jqiikotd/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf1a2405eb1123d1e7a784100d0468a78a7d3f51 --- /dev/null +++ b/jqiikotd/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd59ccb6d36474a69903d07db06a3d41ac461b2f2e01f87aa8a60e5ab21247a +size 1185222 diff --git a/jqiikotd/eval_video/3_eval.mp4 b/jqiikotd/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac6d9e2c5e018680f1c609e200e4d7345265b2e0 --- /dev/null +++ b/jqiikotd/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4804a8e4dbe4f9be433dac9795f084555bc3c7ae419292e533d7ac3fc26dc246 +size 1173915 diff --git a/jqiikotd/eval_video/4_eval.mp4 b/jqiikotd/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b6396cc55e0b6d42fdd8e2d15f1caf8e552acd6 --- /dev/null +++ b/jqiikotd/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3747e786ba19b8c5b5aa57af2419c9b26025875f1c71cd07383b11558ece68f +size 1208718 diff --git a/jqiikotd/tb/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 b/jqiikotd/tb/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 new file mode 100644 index 0000000000000000000000000000000000000000..30c45e73c1ad6362f337a81fc4603752db18be81 --- /dev/null +++ b/jqiikotd/tb/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e679f79a5fd127dd50abe3fb9571913cb8bb95729002e4874346ed66d064a5 +size 6539 diff --git a/jqiikotd/wandb/debug-internal.log b/jqiikotd/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..724f9069882f413ac2af77bd04d43fb1c74b94a4 --- /dev/null +++ b/jqiikotd/wandb/debug-internal.log @@ -0,0 +1,777 @@ +2025-01-07 17:23:07,353 INFO StreamThr :2198748 [internal.py:wandb_internal():85] W&B internal server running at pid: 2198748, started at: 2025-01-07 17:23:07.353456 +2025-01-07 17:23:07,355 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:23:07,366 INFO WriterThread:2198748 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb +2025-01-07 17:23:07,367 DEBUG SenderThread:2198748 [sender.py:send():379] send: header +2025-01-07 17:23:07,375 DEBUG SenderThread:2198748 [sender.py:send():379] send: run +2025-01-07 17:23:07,632 INFO SenderThread:2198748 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files +2025-01-07 17:23:07,632 INFO SenderThread:2198748 [sender.py:_start_run_threads():1188] run started: jqiikotd with start time 1736288587.354733 +2025-01-07 17:23:07,650 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:23:07,650 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:23:07,725 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:23:07,730 DEBUG HandlerThread:2198748 [system_info.py:__init__():26] System info init +2025-01-07 17:23:07,730 DEBUG HandlerThread:2198748 [system_info.py:__init__():41] System info init done +2025-01-07 17:23:07,730 INFO HandlerThread:2198748 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:23:07,730 INFO SystemMonitor:2198748 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:23:07,730 INFO HandlerThread:2198748 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:23:07,730 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:23:07,731 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:23:07,732 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:23:07,732 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:23:07,734 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:23:07,747 DEBUG HandlerThread:2198748 [system_info.py:probe():152] Probing system +2025-01-07 17:23:07,749 DEBUG HandlerThread:2198748 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_info.py:probe():200] Probing system done +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:23:07.747873', 'startedAt': '2025-01-07T22:23:07.340127', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.51675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.974, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.897, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.115, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 756.7562294006348}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:23:07,754 INFO HandlerThread:2198748 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:23:07,754 INFO HandlerThread:2198748 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_info.py:_save_code():44] Saving code +2025-01-07 17:23:07,756 DEBUG HandlerThread:2198748 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:23:07,756 DEBUG HandlerThread:2198748 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:23:07,829 DEBUG HandlerThread:2198748 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:23:07,829 INFO HandlerThread:2198748 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:23:07,833 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:23:07,833 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:23:07,833 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:23:07,833 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:23:08,069 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:23:08,069 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:23:08,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:08,070 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:08,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:08,072 INFO HandlerThread:2198748 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/tb" + save: true +} +control { + relay_id: "f56301adeab34d7f8f7d9c34588d90e0" +} +_info { + stream_id: "jqiikotd" +} + +2025-01-07 17:23:08,073 DEBUG HandlerThread:2198748 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:23:08,085 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/9t7uhj1k-code/FISH/eval_robot.py +2025-01-07 17:23:08,094 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/pi1718ry-wandb-metadata.json +2025-01-07 17:23:08,108 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/jms1b277-diff.patch +2025-01-07 17:23:08,194 DEBUG SenderThread:2198748 [sender.py:send():379] send: telemetry +2025-01-07 17:23:08,194 DEBUG SenderThread:2198748 [sender.py:send():379] send: tbrecord +2025-01-07 17:23:08,195 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:23:08,195 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 with policy live +2025-01-07 17:23:08,368 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/ppjugv7w-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:23:08,635 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json +2025-01-07 17:23:08,635 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch +2025-01-07 17:23:08,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py +2025-01-07 17:23:08,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:08,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:23:08,637 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt +2025-01-07 17:23:08,637 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code +2025-01-07 17:23:08,637 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH +2025-01-07 17:23:09,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:10,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:10,633 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:11,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:12,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:13,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:13,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:14,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:15,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:16,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:16,573 DEBUG SenderThread:2198748 [sender.py:send():379] send: config +2025-01-07 17:23:17,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:18,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:18,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:18,635 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:19,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:20,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:21,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:22,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:22,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:23,069 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:23,070 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:23,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:23,193 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:24,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:25,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:26,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:26,638 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:27,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:28,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:29,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:29,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:30,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:31,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:32,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:33,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:34,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:34,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:35,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:36,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:37,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:38,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:38,070 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:38,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:38,641 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:39,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:39,204 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:39,641 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml +2025-01-07 17:23:40,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:41,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:42,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:42,642 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:43,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:44,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:45,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:45,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:46,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:47,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:48,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:49,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:50,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:51,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:51,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:52,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:53,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:53,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:53,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:54,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:55,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:56,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:56,183 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:57,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:58,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:59,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:00,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:01,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:02,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:02,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:03,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:04,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:05,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:06,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:07,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:07,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:07,734 DEBUG SystemMonitor:2198748 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:24:07,735 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:24:08,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:08,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:08,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:08,301 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:24:08,302 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:24:08,302 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 with policy now +2025-01-07 17:24:08,302 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:24:08,302 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:24:08,303 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:24:08,552 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/wx5cfzc2-media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:24:08,649 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:08,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:24:08,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:24:08,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos +2025-01-07 17:24:08,651 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:24:08,651 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media +2025-01-07 17:24:08,898 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/gdenwg3p-media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:24:09,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:10,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:10,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:11,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:12,079 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:12,529 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:12,651 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:13,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:14,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:15,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:16,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:16,652 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:17,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:18,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:18,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:19,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:20,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:21,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:21,654 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:24:21,890 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/08656ms4-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:24:22,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:22,654 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:23,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:23,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:23,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:23,208 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:24,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:24,655 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:25,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:26,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:26,656 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:27,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:28,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:29,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:29,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:30,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:31,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:32,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:33,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:34,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:34,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:35,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:36,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:37,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:37,736 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:24:38,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:38,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:38,117 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:39,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:39,143 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:40,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:41,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:42,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:43,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:44,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:45,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:45,086 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:46,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:47,088 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:48,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:49,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:50,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:50,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:51,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:52,092 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:52,996 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:24:52,996 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 with policy now +2025-01-07 17:24:52,997 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:24:53,007 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:24:53,007 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:24:53,007 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:24:53,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:53,075 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:53,091 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:53,246 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/igl497ks-media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:24:53,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:24:53,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:24:53,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:24:53,938 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/twfz5pwb-media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:24:54,097 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:54,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:55,096 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:55,203 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:56,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:57,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:58,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:58,666 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:59,099 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:00,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:00,666 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:01,057 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:01,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:02,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:03,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:04,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:05,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:06,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:06,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:07,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:07,736 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:25:08,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:08,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:08,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:09,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:10,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:10,515 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:25:10,515 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:25:10,516 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:25:10,670 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:10,670 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:25:10,670 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:25:11,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:11,737 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:12,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:12,671 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:13,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:14,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:14,671 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:15,107 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:16,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:16,672 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:17,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:17,569 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:18,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:19,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:20,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:21,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:22,106 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:23,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:23,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:23,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:23,148 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:24,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:25,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:26,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:27,108 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:28,108 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:29,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:29,108 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:30,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:31,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:32,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:33,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:34,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:34,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:35,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:36,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:37,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:37,736 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:25:38,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:38,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:38,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:39,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:39,188 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:40,115 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:41,116 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:41,845 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:25:41,845 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 with policy now +2025-01-07 17:25:41,847 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:25:41,848 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:25:41,848 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:25:41,848 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:25:42,085 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/3zx1qn81-media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:25:42,117 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:42,680 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:42,680 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:25:42,680 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:25:42,681 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:25:42,968 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/9g5ahj0g-media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:25:43,127 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:44,124 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:44,580 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:44,681 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:45,125 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:46,135 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:46,682 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:47,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:48,141 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:48,682 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:49,132 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:50,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:50,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:51,131 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:52,132 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:53,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:53,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:53,136 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:54,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:55,135 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:55,139 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:56,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:57,138 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:58,139 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:59,143 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:00,142 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:01,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:01,143 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:02,144 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:03,141 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:04,146 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:05,142 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:06,090 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:06,146 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:07,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:07,737 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:26:08,093 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:08,095 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:08,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:08,364 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:26:08,364 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:26:08,364 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:26:08,689 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:26:08,689 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:26:08,933 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/9c0n89ib-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:26:09,150 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:10,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:10,689 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:11,133 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:11,156 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:12,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:13,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:14,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:14,691 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:15,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:16,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:17,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:17,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:18,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:19,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:20,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:21,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:22,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:23,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:23,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:23,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:23,162 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:24,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:25,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:26,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:27,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:28,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:29,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:29,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:30,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:31,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:32,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:33,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:34,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:35,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:35,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:36,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:37,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:37,737 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:26:38,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:38,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:38,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:39,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:40,147 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:40,171 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:40,699 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:40,764 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:26:40,764 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 with policy now +2025-01-07 17:26:40,769 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:26:40,770 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:26:40,770 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:26:40,772 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:26:41,017 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/26aoquk9-media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:26:41,165 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:41,700 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:26:41,700 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:26:41,701 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:26:41,989 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/hoh1w3s6-media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:26:42,167 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:42,700 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:43,185 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:44,173 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:44,701 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:45,171 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:45,865 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:46,171 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:47,173 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:48,174 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:48,702 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:49,172 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:50,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:51,093 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:51,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:52,182 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:53,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:53,080 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:53,176 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:53,704 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:26:54,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:54,419 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:26:54,420 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:26:54,420 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:26:54,704 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:54,705 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:26:55,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:56,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:56,649 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:56,705 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:57,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:58,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:58,706 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:59,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:00,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:01,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:02,090 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:02,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:03,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:04,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:05,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:06,176 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:07,176 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:07,738 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:27:07,739 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:08,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:27:08,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:27:08,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:09,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:10,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:11,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:12,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:13,089 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:13,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:14,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:15,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:16,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:17,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:18,089 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:18,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:19,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:20,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:21,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:22,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:23,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:27:23,075 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:27:23,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:23,214 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:24,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:25,185 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:25,995 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:27:25,995 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 with policy now +2025-01-07 17:27:25,997 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:27:25,998 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:25,998 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:25,998 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:26,193 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:26,238 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/34a0imsq-media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:26,715 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:26,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:26,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:26,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:27:27,025 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/gj75fw01-media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:27,216 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:28,190 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:28,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:29,097 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:29,201 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:30,206 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:30,717 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:31,196 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:32,200 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:33,199 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:34,110 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:34,207 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:34,718 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:35,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:27:35,179 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:27:35,204 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:35,394 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:27:35,653 INFO wandb-upload_2:2198748 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa09i9ssz +2025-01-07 17:27:35,719 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table +2025-01-07 17:27:35,811 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:27:35,819 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:27:36,004 INFO SenderThread:2198748 [sender.py:send_request_log_artifact():1525] logged artifact run-jqiikotd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkzODUzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwNTI2Nw==', 'latestArtifact': None}} +2025-01-07 17:27:36,004 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:27:36,004 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 17:27:36,007 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:36,007 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,008 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,008 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:36,008 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,009 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,009 DEBUG SenderThread:2198748 [sender.py:send():379] send: telemetry +2025-01-07 17:27:36,009 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,009 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,009 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,010 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,010 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,010 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,010 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,011 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,011 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,011 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,011 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,011 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,012 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,012 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,012 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,012 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,012 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,013 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,013 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,013 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,013 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,014 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,014 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:27:36,193 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/kfil4rc0-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:36,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:36,277 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/f4tt6tmq-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:36,408 INFO wandb-upload_2:2198748 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo23g4zk_ +2025-01-07 17:27:36,455 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/yjoj3yk4-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval +2025-01-07 17:27:36,759 INFO SenderThread:2198748 [sender.py:send_request_log_artifact():1525] logged artifact run-jqiikotd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkzODU1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwNTI3MA==', 'latestArtifact': None}} +2025-01-07 17:27:36,759 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:27:36,759 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json with policy now +2025-01-07 17:27:36,760 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:36,760 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,760 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,947 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/k3gt4nx5-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:37,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:37,224 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/1mi6sc2m-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:37,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:37,738 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:27:38,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:27:38,075 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:27:38,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:38,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:39,142 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:39,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:40,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:41,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:42,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:43,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:43,685 DEBUG SenderThread:2198748 [sender.py:send():379] send: exit +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:send_exit():588] handling runtime: 275 +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:send_exit():594] send defer +2025-01-07 17:27:43,686 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,686 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:27:43,686 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:27:43,687 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,687 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:27:43,687 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:27:43,687 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,687 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:27:43,687 INFO HandlerThread:2198748 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:27:43,687 DEBUG SystemMonitor:2198748 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:27:43,687 DEBUG SystemMonitor:2198748 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:27:43,688 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:27:43,689 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:27:43,723 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:43,743 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:27:43,743 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:27:43,743 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:27:43,743 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,743 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:27:43,743 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:27:43,744 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:27:43,744 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,744 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:27:43,744 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,744 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:27:43,744 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:27:43,744 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,744 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:27:44,723 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:50,473 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:50,474 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:50,474 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:27:50,474 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,475 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:27:50,475 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:50,476 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:50,476 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:27:50,477 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:50,477 DEBUG SenderThread:2198748 [sender.py:send():379] send: summary +2025-01-07 17:27:50,480 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:50,480 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,480 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:27:50,480 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:27:50,480 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:50,480 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:27:50,481 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,481 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:27:50,484 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:50,687 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:50,726 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:50,798 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:27:50,798 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:50,798 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:50,798 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:27:50,798 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,798 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:27:51,688 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:51,726 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml +2025-01-07 17:27:53,806 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:27:53,806 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:53,806 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:53,806 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:27:53,806 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:53,806 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:27:53,806 INFO SenderThread:2198748 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:27:53,809 INFO SenderThread:2198748 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:27:53,810 INFO SenderThread:2198748 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:27:53,821 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:27:53,821 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:53,821 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:27:53,821 DEBUG SenderThread:2198748 [sender.py:send():379] send: artifact +2025-01-07 17:27:54,006 INFO SenderThread:2198748 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:27:54,006 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:54,006 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:27:54,006 INFO SenderThread:2198748 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:27:54,689 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:54,727 INFO SenderThread:2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:54,727 INFO SenderThread:2198748 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files +2025-01-07 17:27:54,727 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml config.yaml +2025-01-07 17:27:54,728 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json wandb-summary.json +2025-01-07 17:27:54,728 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt requirements.txt +2025-01-07 17:27:54,728 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:27:54,731 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:54,731 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log output.log +2025-01-07 17:27:54,731 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch diff.patch +2025-01-07 17:27:54,733 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:27:54,733 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:27:54,733 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:54,734 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:27:54,734 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:27:54,734 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:27:54,737 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:54,737 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:54,737 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:27:54,737 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:54,739 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:54,739 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:27:54,739 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:54,739 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:27:54,739 INFO SenderThread:2198748 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:27:54,906 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt +2025-01-07 17:27:54,909 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml +2025-01-07 17:27:54,925 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:54,947 INFO wandb-upload_4:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:55,048 INFO wandb-upload_3:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/wkjv9ap3-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:55,249 INFO Thread-11 (_thread_body):2198748 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:27:55,249 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,249 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:27:55,249 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,249 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:27:55,250 INFO SenderThread:2198748 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:27:55,251 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:27:55,251 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,251 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:27:55,251 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,251 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:27:55,251 INFO SenderThread:2198748 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:27:55,421 INFO SenderThread:2198748 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:27:55,421 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:27:55,421 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,421 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,422 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:27:55,422 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:27:55,422 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,422 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send():379] send: final +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send():379] send: footer +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,422 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:27:55,423 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:55,423 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:55,423 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:27:55,423 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:55,424 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:27:55,425 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:27:55,425 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:27:55,426 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:55,478 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:55,478 INFO MainThread:2198748 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:27:55,479 INFO MainThread:2198748 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:27:55,479 INFO MainThread:2198748 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:27:55,479 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:27:55,479 INFO HandlerThread:2198748 [handler.py:finish():882] shutting down handler +2025-01-07 17:27:56,424 INFO WriterThread:2198748 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb +2025-01-07 17:27:56,478 INFO SenderThread:2198748 [sender.py:finish():1615] shutting down sender +2025-01-07 17:27:56,478 INFO SenderThread:2198748 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:27:56,478 INFO SenderThread:2198748 [file_pusher.py:join():175] waiting for file pusher diff --git a/jqiikotd/wandb/debug.log b/jqiikotd/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eb0c8b447fa111269b8192f965065abcee9e9ce1 --- /dev/null +++ b/jqiikotd/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Configure stats pid to 2198584 +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/settings +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug.log +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug-internal.log +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd'} +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():619] starting backend +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():623] setting up manager +2025-01-07 17:23:07,352 INFO MainThread:2198584 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:23:07,354 INFO MainThread:2198584 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:23:07,366 INFO MainThread:2198584 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:23:07,373 INFO MainThread:2198584 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:23:07,649 INFO MainThread:2198584 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:23:07,720 INFO MainThread:2198584 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:23:07,720 INFO MainThread:2198584 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:23:08,072 INFO MainThread:2198584 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:23:08,072 INFO MainThread:2198584 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/tb, True +2025-01-07 17:23:16,573 INFO MainThread:2198584 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:27:56,480 WARNING MsgRouterThr:2198584 [router.py:message_loop():77] message_loop has been closed diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7e07b9ad5d6cbeb475ca36f53f1e876c93a5909 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736288587 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1af31b2ff07685aba62f0764c3073e6e9867fe4d --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch @@ -0,0 +1,21 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..6e694c7 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json new file mode 100644 index 0000000000000000000000000000000000000000..293ed08a7657caa290bb6fb2588274bd3f75a5b5 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 45, 0.4444444444444444, 0.27840888504645445, 0.6185835599662125, 0.30912667516711145, 0.5990480035345346, 0.32985768470590726, 0.5639069154730775, 0.34584976735813877, 0.5483370497726087]]} \ No newline at end of file diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8d85b4c6b9a27fc326066ba36bb62c0d381908a9 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c751aa70732cb0acb7db0ecff477db74aa07b780d478d466f4d27e7581e3b7 +size 1171819 diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddd11416756a8e46ba2a39342105bab79d6bc894 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac128dda904a091cb30d97d31d39844ceca216ce6955f12d5233053170f48e0 +size 1178239 diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf1a2405eb1123d1e7a784100d0468a78a7d3f51 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd59ccb6d36474a69903d07db06a3d41ac461b2f2e01f87aa8a60e5ab21247a +size 1185222 diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac6d9e2c5e018680f1c609e200e4d7345265b2e0 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4804a8e4dbe4f9be433dac9795f084555bc3c7ae419292e533d7ac3fc26dc246 +size 1173915 diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b6396cc55e0b6d42fdd8e2d15f1caf8e552acd6 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3747e786ba19b8c5b5aa57af2419c9b26025875f1c71cd07383b11558ece68f +size 1208718 diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..51fde7d4923bf45e368e9d7dbbbbfd03f233f64c --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736288596.573946]: resetting environment +[INFO] [1736288596.577542]: cleared current plan +[INFO] [1736288596.578798]: moving to home +[INFO] [1736288597.581280]: reached home +[INFO] [1736288597.582642]: reset action history +[INFO] [1736288600.291821]: environment reset +Starting episode 0 +[INFO] [1736288600.293463]: resetting environment +[INFO] [1736288600.297424]: cleared current plan +[INFO] [1736288600.298387]: moving to home +[INFO] [1736288601.300626]: reached home +[INFO] [1736288601.302082]: reset action history +[INFO] [1736288604.010834]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736288616.557121]: resetting environment +[INFO] [1736288616.562960]: cleared current plan +[INFO] [1736288616.564184]: moving to home +[INFO] [1736288617.566803]: reached home +[INFO] [1736288617.567995]: reset action history +[INFO] [1736288620.276413]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/episode_rosbags/episode_0_2025-01-07-17-23-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2198584.log +started roslaunch server http://158.130.50.37:44931/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2198852] +process[print_text-2]: started with pid [2198876] +process[rosbag_record-3]: started with pid [2198877] +started bagging! +[INFO] [1736288647.271805]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736288648.035108]: Stored episode 1. +[INFO] [1736288648.035811]: Saving video... +[INFO] [1736288648.258350]: Video saved! +Starting episode 1 +[INFO] [1736288648.271310]: resetting environment +[INFO] [1736288648.283746]: cleared current plan +[INFO] [1736288648.283963]: moving to home +[INFO] [1736288651.485861]: reached home +[INFO] [1736288651.486862]: reset action history +[INFO] [1736288654.195813]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288661.104448]: resetting environment +[INFO] [1736288661.110255]: cleared current plan +[INFO] [1736288661.110457]: moving to home +[INFO] [1736288662.111894]: reached home +[INFO] [1736288662.112300]: reset action history +[INFO] [1736288664.817738]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/episode_rosbags/episode_1_2025-01-07-17-24-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2198584.log +started roslaunch server http://158.130.50.37:44155/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2198996] +process[print_text-5]: started with pid [2198997] +process[rosbag_record-6]: started with pid [2198998] +started bagging! +[INFO] [1736288691.875313]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736288692.651336]: Stored episode 2. +[INFO] [1736288692.651691]: Saving video... +[INFO] [1736288692.966301]: Video saved! +Starting episode 2 +[INFO] [1736288693.008246]: resetting environment +[INFO] [1736288693.026296]: cleared current plan +[INFO] [1736288693.027077]: moving to home +[INFO] [1736288696.236404]: reached home +[INFO] [1736288696.239297]: reset action history +[INFO] [1736288699.014556]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288709.692284]: resetting environment +[INFO] [1736288709.694564]: cleared current plan +[INFO] [1736288709.694715]: moving to home +[INFO] [1736288710.696448]: reached home +[INFO] [1736288710.696952]: reset action history +[INFO] [1736288713.407177]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/episode_rosbags/episode_2_2025-01-07-17-25-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2198584.log +started roslaunch server http://158.130.50.37:34741/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2199134] +process[print_text-8]: started with pid [2199135] +process[rosbag_record-9]: started with pid [2199136] +started bagging! +For topic gripper_width: timestamp difference is -102429179 for nearest: 1736288713978837939 - target: 1736288714081267118 at idx 14 +For topic gripper_width: timestamp difference is -102263204 for nearest: 1736288714178838738 - target: 1736288714281101942 at idx 15 +For topic gripper_width: timestamp difference is -102097908 for nearest: 1736288714378838142 - target: 1736288714480936050 at idx 16 +[INFO] [1736288740.347501]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1736288741.156798]: Stored episode 3. +[INFO] [1736288741.157981]: Saving video... +[INFO] [1736288741.802108]: Video saved! +Starting episode 3 +[INFO] [1736288741.859294]: resetting environment +[WARN] [1736288741.686582]: Plan exhausted +[WARN] [1736288741.725796]: Plan exhausted +[WARN] [1736288741.766123]: Plan exhausted +[WARN] [1736288741.812548]: Plan exhausted +[WARN] [1736288741.847022]: Plan exhausted +[INFO] [1736288741.869566]: cleared current plan +[INFO] [1736288741.871613]: moving to home +[INFO] [1736288744.580117]: reached home +[INFO] [1736288744.581442]: reset action history +[INFO] [1736288747.361464]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288768.069274]: resetting environment +[INFO] [1736288768.083547]: cleared current plan +[INFO] [1736288768.085457]: moving to home +[INFO] [1736288769.091986]: reached home +[INFO] [1736288769.095406]: reset action history +[INFO] [1736288772.001823]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/episode_rosbags/episode_3_2025-01-07-17-26-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2198584.log +started roslaunch server http://158.130.50.37:44371/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2199307] +process[print_text-11]: started with pid [2199308] +process[rosbag_record-12]: started with pid [2199309] +started bagging! +[INFO] [1736288799.311462]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[WARN] [1736288800.648482]: Plan exhausted +[WARN] [1736288800.689439]: Plan exhausted +[WARN] [1736288800.726562]: Plan exhausted +[WARN] [1736288800.766701]: Plan exhausted +[INFO] [1736288800.145028]: Stored episode 4. +[INFO] [1736288800.150842]: Saving video... +[INFO] [1736288800.721431]: Video saved! +Starting episode 4 +[INFO] [1736288800.775921]: resetting environment +[INFO] [1736288800.798851]: cleared current plan +[INFO] [1736288800.802659]: moving to home +[INFO] [1736288803.811768]: reached home +[INFO] [1736288803.821945]: reset action history +[INFO] [1736288806.614485]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736288813.566530]: resetting environment +[INFO] [1736288813.592417]: cleared current plan +[INFO] [1736288813.595335]: moving to home +[INFO] [1736288814.601205]: reached home +[INFO] [1736288814.607164]: reset action history +[INFO] [1736288817.393689]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/episode_rosbags/episode_4_2025-01-07-17-26-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2198584.log +started roslaunch server http://158.130.50.37:37957/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2199454] +process[print_text-14]: started with pid [2199455] +process[rosbag_record-15]: started with pid [2199456] +started bagging! +[INFO] [1736288844.609336]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1736288845.419880]: Stored episode 5. +[INFO] [1736288845.420203]: Saving video... +[INFO] [1736288845.923678]: Video saved! +[WARN] [1736288845.966217]: Plan exhausted +[WARN] [1736288846.005706]: Plan exhausted +[INFO] [1736288845.997549]: resetting environment +[INFO] [1736288846.026293]: cleared current plan +[INFO] [1736288846.028825]: moving to home +[INFO] [1736288849.337564]: reached home +[INFO] [1736288849.343238]: reset action history +[INFO] [1736288852.206877]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e2db90a452a1ab312a4d5a8c22c448c26d8e0dc7 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:23:07.747873", + "startedAt": "2025-01-07T22:23:07.340127", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3935.51675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.978, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.974, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.897, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.115, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 756.7562294006348 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d7bf9e40e0b5a67e16a9138ba4dfdd6706684020 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "88c751aa70732cb0acb7db0ecff477db74aa07b780d478d466f4d27e7581e3b7", "size": 1171819, "path": "media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4"}, "global_step": 665, "_timestamp": 1736288855.7275603, "_runtime": 268.37282729148865, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "3ac128dda904a091cb30d97d31d39844ceca216ce6955f12d5233053170f48e0", "size": 1178239, "path": "media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4"}, "eval/num_success": 0.0, "episode": 45.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "4bd59ccb6d36474a69903d07db06a3d41ac461b2f2e01f87aa8a60e5ab21247a", "size": 1185222, "path": "media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4804a8e4dbe4f9be433dac9795f084555bc3c7ae419292e533d7ac3fc26dc246", "size": 1173915, "path": "media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "c3747e786ba19b8c5b5aa57af2419c9b26025875f1c71cd07383b11558ece68f", "size": 1208718, "path": "media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://ppmjcqbkon8yolkeey6r6fl1wzrb8wib65kz818qececpy2a0qv4jrohm6x8edywaw3sfvz6bd0i4mz969vd5hs8820vhqcu65v2cntnlrae1um9azh0jkb0mi7bcd4i/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://4v474dhccjeji80d0xlspu644kh88cxp83kp383p6vwp5x8utakfek3prmans8l046nk2b09ewubbntohioxy1d3xo1tnyzdrw5tc1766sp9w0e6lmwje6tok9gg0dpe:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 45, "total_success_rate": 0.4444444444444444, "total_success_rate_lb_0.01": 0.27840888504645445, "total_success_rate_ub_0.01": 0.6185835599662125, "total_success_rate_lb_0.025": 0.30912667516711145, "total_success_rate_ub_0.025": 0.5990480035345346, "total_success_rate_lb_0.05": 0.32985768470590726, "total_success_rate_ub_0.05": 0.5639069154730775, "total_success_rate_lb_0.1": 0.34584976735813877, "total_success_rate_ub_0.1": 0.5483370497726087, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "4ee8b1af4261e98151b69a138b648722a7afe53f2074d953442901ac57dbf6ef", "size": 524, "artifact_path": "wandb-client-artifact://9b8g4qsi8s6wkihq8vvzdbx8w43495qi489y8wuorl2e0xp0uonfnbp2jokmsik1jzjs66fdvhzhpem4r9hdsina2iyjbk58xtu0s7mbj4ouu93moc0e5aaofvkxd5gr/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://b1vpumh6ikw2a6rkscgjtncig50421a4qhmanu83kn81vrspfsjzc1pv6i926dwftnvzkwdv23jqgbtm7kc51bpruhxg0yalbf0i68gnya3ewmigd086m3n1ti9wlfvr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 275}, "eval/total_success_rate": 0.4444444477558136, "eval/total_success_rate_lb_0.01": 0.27840888500213623, "eval/total_success_rate_ub_0.01": 0.6185835599899292, "eval/total_success_rate_lb_0.025": 0.30912667512893677, "eval/total_success_rate_ub_0.025": 0.5990480184555054, "eval/total_success_rate_lb_0.05": 0.3298576772212982, "eval/total_success_rate_ub_0.05": 0.5639069080352783, "eval/total_success_rate_lb_0.1": 0.34584978222846985, "eval/total_success_rate_ub_0.1": 0.5483370423316956} \ No newline at end of file diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug-internal.log b/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..724f9069882f413ac2af77bd04d43fb1c74b94a4 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug-internal.log @@ -0,0 +1,777 @@ +2025-01-07 17:23:07,353 INFO StreamThr :2198748 [internal.py:wandb_internal():85] W&B internal server running at pid: 2198748, started at: 2025-01-07 17:23:07.353456 +2025-01-07 17:23:07,355 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:23:07,366 INFO WriterThread:2198748 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb +2025-01-07 17:23:07,367 DEBUG SenderThread:2198748 [sender.py:send():379] send: header +2025-01-07 17:23:07,375 DEBUG SenderThread:2198748 [sender.py:send():379] send: run +2025-01-07 17:23:07,632 INFO SenderThread:2198748 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files +2025-01-07 17:23:07,632 INFO SenderThread:2198748 [sender.py:_start_run_threads():1188] run started: jqiikotd with start time 1736288587.354733 +2025-01-07 17:23:07,650 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:23:07,650 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:23:07,725 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:23:07,730 DEBUG HandlerThread:2198748 [system_info.py:__init__():26] System info init +2025-01-07 17:23:07,730 DEBUG HandlerThread:2198748 [system_info.py:__init__():41] System info init done +2025-01-07 17:23:07,730 INFO HandlerThread:2198748 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:23:07,730 INFO SystemMonitor:2198748 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:23:07,730 INFO HandlerThread:2198748 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:23:07,730 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:23:07,731 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:23:07,732 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:23:07,732 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:23:07,734 INFO SystemMonitor:2198748 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:23:07,747 DEBUG HandlerThread:2198748 [system_info.py:probe():152] Probing system +2025-01-07 17:23:07,749 DEBUG HandlerThread:2198748 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_info.py:probe():200] Probing system done +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:23:07.747873', 'startedAt': '2025-01-07T22:23:07.340127', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.51675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.974, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.897, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.115, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 756.7562294006348}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:23:07,754 INFO HandlerThread:2198748 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:23:07,754 INFO HandlerThread:2198748 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:23:07,754 DEBUG HandlerThread:2198748 [system_info.py:_save_code():44] Saving code +2025-01-07 17:23:07,756 DEBUG HandlerThread:2198748 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:23:07,756 DEBUG HandlerThread:2198748 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:23:07,829 DEBUG HandlerThread:2198748 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:23:07,829 INFO HandlerThread:2198748 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:23:07,833 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:23:07,833 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:23:07,833 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:23:07,833 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:23:08,069 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:23:08,069 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:23:08,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:08,070 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:08,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:08,072 INFO HandlerThread:2198748 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/tb" + save: true +} +control { + relay_id: "f56301adeab34d7f8f7d9c34588d90e0" +} +_info { + stream_id: "jqiikotd" +} + +2025-01-07 17:23:08,073 DEBUG HandlerThread:2198748 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:23:08,085 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/9t7uhj1k-code/FISH/eval_robot.py +2025-01-07 17:23:08,094 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/pi1718ry-wandb-metadata.json +2025-01-07 17:23:08,108 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/jms1b277-diff.patch +2025-01-07 17:23:08,194 DEBUG SenderThread:2198748 [sender.py:send():379] send: telemetry +2025-01-07 17:23:08,194 DEBUG SenderThread:2198748 [sender.py:send():379] send: tbrecord +2025-01-07 17:23:08,195 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:23:08,195 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 with policy live +2025-01-07 17:23:08,368 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/ppjugv7w-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:23:08,635 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json +2025-01-07 17:23:08,635 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch +2025-01-07 17:23:08,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py +2025-01-07 17:23:08,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:08,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:23:08,637 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt +2025-01-07 17:23:08,637 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code +2025-01-07 17:23:08,637 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH +2025-01-07 17:23:09,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:10,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:10,633 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:11,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:12,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:13,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:13,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:14,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:15,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:16,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:16,573 DEBUG SenderThread:2198748 [sender.py:send():379] send: config +2025-01-07 17:23:17,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:18,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:18,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:18,635 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:19,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:20,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:21,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:22,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:22,636 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:23,069 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:23,070 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:23,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:23,193 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:24,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:25,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:26,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:26,638 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:27,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:28,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:29,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:29,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:30,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:31,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:32,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:33,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:34,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:34,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:35,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:36,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:37,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:38,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:38,070 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:38,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:38,641 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:39,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:39,204 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:39,641 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml +2025-01-07 17:23:40,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:41,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:42,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:42,642 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:23:43,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:44,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:45,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:45,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:46,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:47,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:48,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:49,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:50,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:51,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:51,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:52,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:53,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:23:53,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:23:53,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:54,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:55,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:56,076 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:56,183 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:23:57,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:58,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:23:59,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:00,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:01,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:02,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:02,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:03,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:04,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:05,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:06,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:07,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:07,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:07,734 DEBUG SystemMonitor:2198748 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:24:07,735 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:24:08,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:08,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:08,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:08,301 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:24:08,302 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:24:08,302 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 with policy now +2025-01-07 17:24:08,302 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:24:08,302 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:24:08,303 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:24:08,552 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/wx5cfzc2-media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:24:08,649 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:08,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:24:08,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:24:08,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos +2025-01-07 17:24:08,651 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:24:08,651 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media +2025-01-07 17:24:08,898 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/gdenwg3p-media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:24:09,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:10,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:10,650 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:11,078 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:12,079 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:12,529 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:12,651 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:13,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:14,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:15,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:16,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:16,652 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:17,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:18,072 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:18,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:19,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:20,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:21,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:21,654 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:24:21,890 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/08656ms4-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:24:22,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:22,654 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:23,070 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:23,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:23,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:23,208 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:24,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:24,655 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:25,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:26,081 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:26,656 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:27,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:28,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:29,073 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:29,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:30,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:31,082 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:32,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:33,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:34,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:34,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:35,083 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:36,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:37,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:37,736 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:24:38,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:38,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:38,117 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:39,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:39,143 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:40,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:41,084 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:42,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:43,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:44,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:45,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:45,086 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:46,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:47,088 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:48,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:49,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:50,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:50,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:51,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:52,092 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:52,996 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:24:52,996 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 with policy now +2025-01-07 17:24:52,997 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:24:53,007 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:24:53,007 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:24:53,007 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:24:53,075 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:24:53,075 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:24:53,091 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:53,246 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/igl497ks-media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:24:53,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:24:53,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:24:53,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:24:53,938 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/twfz5pwb-media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:24:54,097 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:54,664 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:55,096 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:55,203 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:24:56,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:57,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:58,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:24:58,666 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:24:59,099 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:00,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:00,666 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:01,057 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:01,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:02,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:03,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:04,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:05,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:06,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:06,100 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:07,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:07,736 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:25:08,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:08,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:08,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:09,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:10,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:10,515 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:25:10,515 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:25:10,516 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:25:10,670 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:10,670 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:25:10,670 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:25:11,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:11,737 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:12,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:12,671 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:13,101 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:14,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:14,671 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:15,107 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:16,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:16,672 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:17,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:17,569 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:18,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:19,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:20,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:21,102 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:22,106 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:23,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:23,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:23,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:23,148 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:24,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:25,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:26,103 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:27,108 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:28,108 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:29,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:29,108 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:30,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:31,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:32,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:33,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:34,077 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:34,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:35,113 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:36,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:37,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:37,736 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:25:38,071 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:38,071 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:38,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:39,114 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:39,188 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:40,115 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:41,116 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:41,845 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:25:41,845 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 with policy now +2025-01-07 17:25:41,847 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:25:41,848 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:25:41,848 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:25:41,848 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:25:42,085 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/3zx1qn81-media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:25:42,117 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:42,680 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:42,680 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:25:42,680 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:25:42,681 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:25:42,968 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/9g5ahj0g-media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:25:43,127 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:44,124 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:44,580 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:44,681 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:45,125 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:46,135 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:46,682 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:47,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:48,141 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:48,682 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:25:49,132 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:50,087 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:50,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:51,131 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:52,132 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:53,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:25:53,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:25:53,136 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:54,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:55,135 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:55,139 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:25:56,134 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:57,138 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:58,139 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:25:59,143 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:00,142 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:01,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:01,143 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:02,144 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:03,141 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:04,146 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:05,142 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:06,090 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:06,146 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:07,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:07,737 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:26:08,093 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:08,095 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:08,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:08,364 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:26:08,364 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:26:08,364 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:26:08,689 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:26:08,689 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:26:08,933 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/9c0n89ib-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:26:09,150 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:10,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:10,689 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:11,133 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:11,156 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:12,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:13,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:14,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:14,691 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:15,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:16,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:17,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:17,151 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:18,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:19,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:20,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:21,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:22,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:23,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:23,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:23,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:23,162 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:24,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:25,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:26,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:27,152 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:28,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:29,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:29,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:30,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:31,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:32,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:33,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:34,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:35,085 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:35,153 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:36,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:37,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:37,737 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:26:38,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:38,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:38,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:39,154 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:40,147 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:40,171 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:40,699 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:40,764 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:26:40,764 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 with policy now +2025-01-07 17:26:40,769 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:26:40,770 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:26:40,770 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:26:40,772 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:26:41,017 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/26aoquk9-media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:26:41,165 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:41,700 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:26:41,700 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:26:41,701 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:26:41,989 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/hoh1w3s6-media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:26:42,167 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:42,700 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:43,185 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:44,173 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:44,701 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:45,171 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:45,865 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:46,171 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:47,173 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:48,174 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:48,702 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:49,172 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:50,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:51,093 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:51,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:52,182 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:53,080 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:26:53,080 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:26:53,176 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:53,704 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:26:54,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:54,419 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:26:54,420 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:26:54,420 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:26:54,704 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:54,705 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:26:55,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:56,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:56,649 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:26:56,705 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:57,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:58,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:26:58,706 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:26:59,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:00,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:01,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:02,090 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:02,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:03,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:04,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:05,175 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:06,176 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:07,176 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:07,738 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:27:07,739 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:08,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:27:08,074 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:27:08,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:09,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:10,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:11,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:12,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:13,089 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:13,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:14,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:15,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:16,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:17,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:18,089 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:18,177 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:19,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:20,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:21,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:22,178 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:23,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:27:23,075 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:27:23,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:23,214 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:24,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:25,185 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:25,995 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:27:25,995 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 with policy now +2025-01-07 17:27:25,997 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:27:25,998 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:25,998 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:25,998 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:26,193 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:26,238 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/34a0imsq-media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:26,715 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:26,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:26,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:26,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval +2025-01-07 17:27:27,025 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/gj75fw01-media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:27,216 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:28,190 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:28,716 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:29,097 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:29,201 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:30,206 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:30,717 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:31,196 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:32,200 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:33,199 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:34,110 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:34,207 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:34,718 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:35,179 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:27:35,179 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:27:35,204 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:35,394 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:27:35,653 INFO wandb-upload_2:2198748 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa09i9ssz +2025-01-07 17:27:35,719 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval +2025-01-07 17:27:35,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table +2025-01-07 17:27:35,811 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:27:35,819 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:27:36,004 INFO SenderThread:2198748 [sender.py:send_request_log_artifact():1525] logged artifact run-jqiikotd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkzODUzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwNTI2Nw==', 'latestArtifact': None}} +2025-01-07 17:27:36,004 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:27:36,004 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 17:27:36,007 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:36,007 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,008 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,008 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:36,008 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,009 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,009 DEBUG SenderThread:2198748 [sender.py:send():379] send: telemetry +2025-01-07 17:27:36,009 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,009 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,009 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,010 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,010 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,010 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,010 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,011 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,011 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,011 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,011 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,011 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,012 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,012 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,012 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,012 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,012 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,013 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,013 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,013 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,013 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,014 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,014 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:27:36,193 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/kfil4rc0-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:36,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:36,277 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/f4tt6tmq-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:36,408 INFO wandb-upload_2:2198748 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo23g4zk_ +2025-01-07 17:27:36,455 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/yjoj3yk4-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:36,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval +2025-01-07 17:27:36,759 INFO SenderThread:2198748 [sender.py:send_request_log_artifact():1525] logged artifact run-jqiikotd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkzODU1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcwNTI3MA==', 'latestArtifact': None}} +2025-01-07 17:27:36,759 DEBUG SenderThread:2198748 [sender.py:send():379] send: files +2025-01-07 17:27:36,759 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json with policy now +2025-01-07 17:27:36,760 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:36,760 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:36,760 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:36,947 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/k3gt4nx5-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:37,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:37,224 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/1mi6sc2m-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:37,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:37,738 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:27:38,074 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:27:38,075 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:27:38,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:38,720 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:39,142 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:39,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:40,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:41,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:42,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:43,198 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:43,685 DEBUG SenderThread:2198748 [sender.py:send():379] send: exit +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:send_exit():588] handling runtime: 275 +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:43,686 INFO SenderThread:2198748 [sender.py:send_exit():594] send defer +2025-01-07 17:27:43,686 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,686 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:27:43,686 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:27:43,687 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,687 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:27:43,687 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:27:43,687 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:27:43,687 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,687 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:27:43,687 INFO HandlerThread:2198748 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:27:43,687 DEBUG SystemMonitor:2198748 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:27:43,687 DEBUG SystemMonitor:2198748 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:27:43,688 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:27:43,689 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:27:43,723 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:43,743 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:27:43,743 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:27:43,743 INFO HandlerThread:2198748 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:27:43,743 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,743 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:27:43,743 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:27:43,744 DEBUG SenderThread:2198748 [sender.py:send():379] send: stats +2025-01-07 17:27:43,744 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,744 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:27:43,744 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:43,744 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:27:43,744 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:27:43,744 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:43,744 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:27:44,723 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:50,473 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:50,474 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:50,474 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:27:50,474 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,475 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:27:50,475 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send():379] send: history +2025-01-07 17:27:50,475 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:27:50,476 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:50,476 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:27:50,477 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:50,477 DEBUG SenderThread:2198748 [sender.py:send():379] send: summary +2025-01-07 17:27:50,480 INFO SenderThread:2198748 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:27:50,480 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,480 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:27:50,480 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:27:50,480 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:50,480 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:27:50,481 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,481 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:27:50,484 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:27:50,687 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:50,726 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:50,798 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:27:50,798 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:50,798 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:50,798 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:27:50,798 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:50,798 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:27:51,688 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:51,726 INFO Thread-12 :2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml +2025-01-07 17:27:53,806 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:27:53,806 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:53,806 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:53,806 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:27:53,806 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:53,806 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:27:53,806 INFO SenderThread:2198748 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:27:53,809 INFO SenderThread:2198748 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:27:53,810 INFO SenderThread:2198748 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:27:53,821 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:27:53,821 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:53,821 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:27:53,821 DEBUG SenderThread:2198748 [sender.py:send():379] send: artifact +2025-01-07 17:27:54,006 INFO SenderThread:2198748 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:27:54,006 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:54,006 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:27:54,006 INFO SenderThread:2198748 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:27:54,689 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:54,727 INFO SenderThread:2198748 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:54,727 INFO SenderThread:2198748 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files +2025-01-07 17:27:54,727 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml config.yaml +2025-01-07 17:27:54,728 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json wandb-summary.json +2025-01-07 17:27:54,728 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt requirements.txt +2025-01-07 17:27:54,728 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:27:54,731 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:54,731 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log output.log +2025-01-07 17:27:54,731 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/diff.patch diff.patch +2025-01-07 17:27:54,733 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:27:54,733 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 +2025-01-07 17:27:54,733 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 +2025-01-07 17:27:54,734 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 +2025-01-07 17:27:54,734 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 +2025-01-07 17:27:54,734 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 +2025-01-07 17:27:54,737 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:27:54,737 INFO SenderThread:2198748 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:27:54,737 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:27:54,737 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:54,739 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:54,739 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:27:54,739 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:54,739 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:27:54,739 INFO SenderThread:2198748 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:27:54,906 INFO wandb-upload_0:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/requirements.txt +2025-01-07 17:27:54,909 INFO wandb-upload_1:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/config.yaml +2025-01-07 17:27:54,925 INFO wandb-upload_2:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/wandb-summary.json +2025-01-07 17:27:54,947 INFO wandb-upload_4:2198748 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/files/output.log +2025-01-07 17:27:55,048 INFO wandb-upload_3:2198748 [upload_job.py:push():130] Uploaded file /tmp/tmpl1__zpe1wandb/wkjv9ap3-events.out.tfevents.1736288588.leonmkim-ROG-Strix-G15CS-G15CS.2198584.0 +2025-01-07 17:27:55,249 INFO Thread-11 (_thread_body):2198748 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:27:55,249 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,249 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:27:55,249 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,249 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:27:55,250 INFO SenderThread:2198748 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:27:55,251 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:27:55,251 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,251 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:27:55,251 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,251 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:27:55,251 INFO SenderThread:2198748 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:27:55,421 INFO SenderThread:2198748 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:27:55,421 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:27:55,421 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,421 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,422 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:27:55,422 INFO SenderThread:2198748 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:27:55,422 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:27:55,422 INFO HandlerThread:2198748 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send():379] send: final +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send():379] send: footer +2025-01-07 17:27:55,422 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: defer +2025-01-07 17:27:55,422 INFO SenderThread:2198748 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:27:55,423 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:55,423 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:55,423 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:27:55,423 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:27:55,424 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:27:55,425 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:27:55,425 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:27:55,426 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:27:55,478 DEBUG SenderThread:2198748 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:27:55,478 INFO MainThread:2198748 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:27:55,479 INFO MainThread:2198748 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:27:55,479 INFO MainThread:2198748 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:27:55,479 DEBUG HandlerThread:2198748 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:27:55,479 INFO HandlerThread:2198748 [handler.py:finish():882] shutting down handler +2025-01-07 17:27:56,424 INFO WriterThread:2198748 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb +2025-01-07 17:27:56,478 INFO SenderThread:2198748 [sender.py:finish():1615] shutting down sender +2025-01-07 17:27:56,478 INFO SenderThread:2198748 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:27:56,478 INFO SenderThread:2198748 [file_pusher.py:join():175] waiting for file pusher diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug.log b/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eb0c8b447fa111269b8192f965065abcee9e9ce1 --- /dev/null +++ b/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Configure stats pid to 2198584 +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/settings +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:23:07,347 INFO MainThread:2198584 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug.log +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/wandb/run-20250107_172307-jqiikotd/logs/debug-internal.log +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd'} +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():619] starting backend +2025-01-07 17:23:07,348 INFO MainThread:2198584 [wandb_init.py:init():623] setting up manager +2025-01-07 17:23:07,352 INFO MainThread:2198584 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:23:07,354 INFO MainThread:2198584 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:23:07,366 INFO MainThread:2198584 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:23:07,373 INFO MainThread:2198584 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:23:07,649 INFO MainThread:2198584 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:23:07,720 INFO MainThread:2198584 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:23:07,720 INFO MainThread:2198584 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:23:08,070 INFO MainThread:2198584 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:23:08,072 INFO MainThread:2198584 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:23:08,072 INFO MainThread:2198584 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/jqiikotd/tb, True +2025-01-07 17:23:16,573 INFO MainThread:2198584 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:27:56,480 WARNING MsgRouterThr:2198584 [router.py:message_loop():77] message_loop has been closed diff --git a/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb b/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ecc047837bd7dc17337eb19473ff633848f28e28 Binary files /dev/null and b/jqiikotd/wandb/run-20250107_172307-jqiikotd/run-jqiikotd.wandb differ diff --git a/you6ychq/episode_rosbags/episode_1_2025-01-07-17-42-15.bag b/you6ychq/episode_rosbags/episode_1_2025-01-07-17-42-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..17d6782d76e16c81c8ab4f8a86c902a8bbd3246d --- /dev/null +++ b/you6ychq/episode_rosbags/episode_1_2025-01-07-17-42-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac0a511aabe1d3fd4c0ffc6a3fb23b5b0e04a3ef3d938a4fcb85e878883ca5f9 +size 1413861337 diff --git a/yyf8aq3v/.hydra/config.yaml b/yyf8aq3v/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/yyf8aq3v/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/yyf8aq3v/.hydra/hydra.yaml b/yyf8aq3v/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10d0039e5fdfcbc6ebc6c68cce564a0ff1c66d05 --- /dev/null +++ b/yyf8aq3v/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/yyf8aq3v/.hydra/overrides.yaml b/yyf8aq3v/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/yyf8aq3v/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/yyf8aq3v/episode_rosbags/episode_0_2025-01-07-17-37-08.bag b/yyf8aq3v/episode_rosbags/episode_0_2025-01-07-17-37-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..fedb684bd42a0668515215e1d172bdc47f5aabb7 --- /dev/null +++ b/yyf8aq3v/episode_rosbags/episode_0_2025-01-07-17-37-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe8693e62a977bfb2e5899f47ad893776bd2d741433b144d1dae132f73a8f14 +size 1410175230 diff --git a/yyf8aq3v/episode_rosbags/episode_1_2025-01-07-17-37-48.bag b/yyf8aq3v/episode_rosbags/episode_1_2025-01-07-17-37-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..da6251262db53334bce7e158d19ff9fab0c53f23 --- /dev/null +++ b/yyf8aq3v/episode_rosbags/episode_1_2025-01-07-17-37-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c0298e7cca41c45897a81b325c2f7b9223d58b7a135a955f556a312e74179e8 +size 1410186958 diff --git a/yyf8aq3v/episode_rosbags/episode_2_2025-01-07-17-38-32.bag b/yyf8aq3v/episode_rosbags/episode_2_2025-01-07-17-38-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..e002a08671679e1839d6e22bf68174d0d501d08d --- /dev/null +++ b/yyf8aq3v/episode_rosbags/episode_2_2025-01-07-17-38-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53ed9e8fb88cf59a201c16ac814ea3c7505b9284e68ceabfa04eb93b1c7f850e +size 1412911859 diff --git a/yyf8aq3v/eval_video/0_eval.mp4 b/yyf8aq3v/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..465c6f1dc5b30088270d1ccee09c25ac68c833d7 --- /dev/null +++ b/yyf8aq3v/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90026e2343d0351af69a1f8a7c0e73e2229ceb186ccb6bfa557a8be5278e2132 +size 1148153 diff --git a/yyf8aq3v/eval_video/1_eval.mp4 b/yyf8aq3v/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef54867939d020f76e5b1dce3ac9678d5a724a3d --- /dev/null +++ b/yyf8aq3v/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:269892ed51d4f6e3e256c71742e308af88e94e3950f5cb2af75013a41e0809f4 +size 1152547 diff --git a/yyf8aq3v/eval_video/2_eval.mp4 b/yyf8aq3v/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8ef50361199ea6d8f40f8059e44ad93e54d4869 --- /dev/null +++ b/yyf8aq3v/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651847148ddc37157f47c67bf89d6ec691eb37886c34e76041adfdf18b3e3aea +size 1130336 diff --git a/yyf8aq3v/eval_video/3_eval.mp4 b/yyf8aq3v/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..050aab2ce2b5bd36e02f36568b80eb07481ba3fa --- /dev/null +++ b/yyf8aq3v/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e672402e025995fe93eba5820cc297d120d4275184b78a82d6ad559304d292f +size 1144841 diff --git a/yyf8aq3v/eval_video/4_eval.mp4 b/yyf8aq3v/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b54a0f7dec6c9aff2749d2c4c48961175442fa7 --- /dev/null +++ b/yyf8aq3v/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1fe43311f51687c2c7a14e0e7261f20b724a3e9c42ef4004f5d9f7073493a6f +size 1223808 diff --git a/yyf8aq3v/tb/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 b/yyf8aq3v/tb/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 new file mode 100644 index 0000000000000000000000000000000000000000..4b18cfa9f0f6897941f4a0b82acb2b5a23be539c --- /dev/null +++ b/yyf8aq3v/tb/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd59d48af0c8ecf44977361d4d8885eda56f0c8352407dc835de1bca9ec3482 +size 6539 diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug-internal.log b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c68bc2ee9ea608b12f3d430cf6db9d1113f3e9e0 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug-internal.log @@ -0,0 +1,752 @@ +2025-01-07 17:36:22,061 INFO StreamThr :2200445 [internal.py:wandb_internal():85] W&B internal server running at pid: 2200445, started at: 2025-01-07 17:36:22.061446 +2025-01-07 17:36:22,063 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:36:22,074 INFO WriterThread:2200445 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb +2025-01-07 17:36:22,075 DEBUG SenderThread:2200445 [sender.py:send():379] send: header +2025-01-07 17:36:22,082 DEBUG SenderThread:2200445 [sender.py:send():379] send: run +2025-01-07 17:36:22,320 INFO SenderThread:2200445 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files +2025-01-07 17:36:22,320 INFO SenderThread:2200445 [sender.py:_start_run_threads():1188] run started: yyf8aq3v with start time 1736289382.061578 +2025-01-07 17:36:22,336 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:36:22,336 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:36:22,409 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:36:22,414 DEBUG HandlerThread:2200445 [system_info.py:__init__():26] System info init +2025-01-07 17:36:22,414 DEBUG HandlerThread:2200445 [system_info.py:__init__():41] System info init done +2025-01-07 17:36:22,414 INFO HandlerThread:2200445 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:36:22,414 INFO SystemMonitor:2200445 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:36:22,415 INFO HandlerThread:2200445 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:36:22,415 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:36:22,415 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:36:22,416 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:36:22,417 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:36:22,417 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:36:22,427 DEBUG HandlerThread:2200445 [system_info.py:probe():152] Probing system +2025-01-07 17:36:22,429 DEBUG HandlerThread:2200445 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_info.py:probe():200] Probing system done +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:36:22.427733', 'startedAt': '2025-01-07T22:36:22.049890', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.23825, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.272, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.943, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 763.361328125}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:36:22,433 INFO HandlerThread:2200445 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:36:22,433 INFO HandlerThread:2200445 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_info.py:_save_code():44] Saving code +2025-01-07 17:36:22,439 DEBUG HandlerThread:2200445 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:36:22,439 DEBUG HandlerThread:2200445 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:36:22,501 DEBUG HandlerThread:2200445 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:36:22,502 INFO HandlerThread:2200445 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:36:22,505 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:36:22,505 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:36:22,505 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:36:22,505 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:36:22,751 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/3jo5rt4f-diff.patch +2025-01-07 17:36:22,757 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/tgxibp13-wandb-metadata.json +2025-01-07 17:36:22,759 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/474yrsfp-code/FISH/eval_robot.py +2025-01-07 17:36:22,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:36:22,759 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:36:22,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:36:22,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:36:22,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:22,762 INFO HandlerThread:2200445 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/tb" + save: true +} +control { + relay_id: "b694256fd9024681b0cafe8a5a7a7ed6" +} +_info { + stream_id: "yyf8aq3v" +} + +2025-01-07 17:36:22,762 DEBUG HandlerThread:2200445 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:36:22,833 DEBUG SenderThread:2200445 [sender.py:send():379] send: telemetry +2025-01-07 17:36:22,833 DEBUG SenderThread:2200445 [sender.py:send():379] send: tbrecord +2025-01-07 17:36:22,834 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:36:22,834 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 with policy live +2025-01-07 17:36:23,025 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/3esqj776-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:36:23,321 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code +2025-01-07 17:36:23,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:24,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:25,321 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:25,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:26,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:27,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:27,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:28,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:29,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:30,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:31,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:32,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:33,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:33,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:34,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:35,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:36,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:37,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:36:37,759 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:36:37,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:38,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:38,911 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:39,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:40,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:41,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:42,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:43,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:44,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:44,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:45,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:46,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:47,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:48,115 DEBUG SenderThread:2200445 [sender.py:send():379] send: config +2025-01-07 17:36:48,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:49,327 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:49,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:50,165 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:50,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:51,328 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:51,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:52,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:36:52,759 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:36:52,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:53,328 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:53,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:54,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:55,329 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:55,556 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:55,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:56,329 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml +2025-01-07 17:36:56,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:57,330 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:57,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:58,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:59,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:00,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:00,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:01,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:02,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:03,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:04,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:05,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:06,257 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:06,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:07,332 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:07,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:07,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:07,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:08,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:09,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:10,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:11,333 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:11,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:12,243 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:12,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:13,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:14,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:15,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:16,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:17,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:17,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:18,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:19,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:20,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:21,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:22,417 DEBUG SystemMonitor:2200445 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:37:22,418 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:37:22,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:22,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:22,805 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:22,892 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:23,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:24,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:25,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:26,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:27,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:28,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:28,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:29,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:30,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:31,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:32,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:33,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:33,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:34,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:35,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:36,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:36,941 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:37:36,942 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:37:36,942 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 with policy now +2025-01-07 17:37:36,942 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:37:36,943 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:37:36,943 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:37:37,162 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/kfxfaftv-media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:37:37,615 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/8oax5jws-media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:37:37,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:37,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:37,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:38,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:38,877 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:39,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:39,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:40,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:41,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:42,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:43,342 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:43,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:44,377 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:44,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:45,343 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:37:45,343 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:45,584 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/wzto8sho-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:37:45,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:46,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:47,344 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:47,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:48,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:49,779 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:49,837 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:50,779 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:51,345 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:51,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:52,419 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:37:52,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:52,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:52,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:53,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:54,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:54,856 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:55,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:56,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:57,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:58,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:59,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:00,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:00,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:01,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:02,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:03,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:04,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:05,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:05,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:06,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:07,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:07,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:07,805 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:08,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:09,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:10,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:10,886 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:11,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:12,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:13,353 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:13,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:14,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:15,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:16,252 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:16,628 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:38:16,631 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:38:16,631 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 with policy now +2025-01-07 17:38:16,632 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:38:16,632 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:38:16,632 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:38:16,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:16,851 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/u0w1hrt2-media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:38:17,354 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:38:17,354 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:17,355 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:38:17,355 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:38:17,629 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/fo2gg6ut-media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:38:17,788 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:18,785 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:19,355 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:19,794 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:20,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:21,356 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:21,405 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:21,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:22,419 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:38:22,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:22,764 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:22,809 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:23,786 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:24,786 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:25,357 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:25,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:26,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:27,178 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:27,788 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:28,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:29,184 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:38:29,184 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:38:29,184 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:38:29,359 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:38:29,359 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:38:29,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:30,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:31,359 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:31,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:32,312 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:32,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:33,360 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:33,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:34,790 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:35,361 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:35,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:36,790 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:37,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:37,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:37,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:37,872 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:38,790 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:39,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:40,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:41,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:42,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:43,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:43,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:44,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:45,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:46,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:47,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:48,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:49,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:49,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:50,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:51,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:52,419 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:38:52,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:52,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:52,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:53,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:54,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:54,876 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:55,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:56,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:57,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:58,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:59,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:59,877 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:59,917 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:38:59,917 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:38:59,918 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 with policy now +2025-01-07 17:38:59,918 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:38:59,918 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:38:59,918 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:00,138 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/fizdclx7-media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:39:00,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:00,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:39:00,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:39:00,641 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/67c7iyip-media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:39:00,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:01,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:01,797 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:02,802 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:03,798 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:04,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:05,371 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:05,685 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:05,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:06,799 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:07,371 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:07,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:07,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:07,809 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:08,015 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:39:08,015 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:39:08,015 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:08,372 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:39:08,372 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:08,619 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/uc69riqb-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:39:08,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:09,372 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:09,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:10,802 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:10,949 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:11,373 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:11,830 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:12,820 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:13,373 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:13,821 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:14,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:15,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:16,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:16,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:17,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:18,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:19,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:20,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:21,773 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:21,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:22,420 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:39:22,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:22,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:22,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:23,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:24,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:25,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:26,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:26,840 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:27,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:28,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:29,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:30,824 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:31,824 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:32,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:32,824 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:33,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:34,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:35,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:36,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:37,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:37,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:37,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:37,903 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:38,833 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:39,828 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:40,287 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:39:40,288 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 with policy now +2025-01-07 17:39:40,294 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:39:40,294 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:39:40,294 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:39:40,295 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:40,382 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:40,382 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:39:40,383 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:39:40,519 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/hzk97aqp-media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:39:40,794 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/1n1ct8aa-media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:39:40,831 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:41,383 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:41,835 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:42,383 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:42,831 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:43,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:43,836 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:44,839 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:45,384 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:45,835 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:46,873 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:47,841 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:48,773 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:48,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:49,385 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:49,840 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:50,386 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:39:50,836 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:51,070 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:39:51,070 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:39:51,070 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:51,386 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:51,386 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:51,847 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:52,420 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:39:52,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:52,763 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:52,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:53,387 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:53,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:53,869 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:54,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:55,388 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:55,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:56,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:57,389 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:57,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:58,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:59,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:59,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:00,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:01,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:02,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:03,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:04,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:04,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:05,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:06,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:07,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:40:07,762 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:40:07,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:08,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:09,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:09,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:10,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:11,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:12,845 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:13,845 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:14,845 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:15,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:15,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:16,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:17,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:18,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:19,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:20,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:20,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:21,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:22,421 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:40:22,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:40:22,762 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:40:22,847 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:23,849 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:24,143 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:40:24,143 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 with policy now +2025-01-07 17:40:24,148 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:40:24,149 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:24,149 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:24,149 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:24,367 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/tihhbsaj-media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:24,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:24,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:24,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:40:24,679 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/v30vykpu-media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:24,854 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:25,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:25,858 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:26,177 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:26,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:26,855 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:27,864 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:28,858 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:29,400 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:29,866 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:30,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:31,733 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:31,863 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:32,118 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:32,119 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:32,142 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:32,401 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:32,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:32,597 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:40:32,598 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:40:32,663 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/4z5hm4g1-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:32,839 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:40:32,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:33,047 INFO wandb-upload_2:2200445 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprjzsorss +2025-01-07 17:40:33,293 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:40:33,304 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media +2025-01-07 17:40:33,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval +2025-01-07 17:40:33,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table +2025-01-07 17:40:33,584 INFO SenderThread:2200445 [sender.py:send_request_log_artifact():1525] logged artifact run-yyf8aq3v-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk3Nzg0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxNTgyNw==', 'latestArtifact': None}} +2025-01-07 17:40:33,586 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:40:33,586 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 17:40:33,587 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:33,587 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,591 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,592 DEBUG SenderThread:2200445 [sender.py:send():379] send: telemetry +2025-01-07 17:40:33,592 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,598 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,610 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,620 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,620 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,771 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/b0zt5kaw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:33,773 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,773 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,792 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,792 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,798 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/kauhvnds-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:33,808 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,808 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,817 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,818 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,820 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,820 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,824 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,824 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,826 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,826 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,829 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,830 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,832 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,832 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:40:33,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:34,033 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/cisms8s4-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:34,223 INFO wandb-upload_1:2200445 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk4pfvn46 +2025-01-07 17:40:34,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:34,615 INFO SenderThread:2200445 [sender.py:send_request_log_artifact():1525] logged artifact run-yyf8aq3v-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk3Nzk4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxNTkxOA==', 'latestArtifact': None}} +2025-01-07 17:40:34,615 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:40:34,615 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json with policy now +2025-01-07 17:40:34,615 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:34,616 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:34,616 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:34,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:34,888 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/pq63jbqt-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:35,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:35,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:35,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:36,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:37,617 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:37,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:40:37,762 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:40:37,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:38,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:39,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:40,877 DEBUG SenderThread:2200445 [sender.py:send():379] send: exit +2025-01-07 17:40:40,877 INFO SenderThread:2200445 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:send_exit():588] handling runtime: 258 +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:send_exit():594] send defer +2025-01-07 17:40:40,878 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,878 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:40:40,878 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:40:40,879 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:40:40,879 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,879 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:40:40,879 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:40:40,879 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:40:40,880 DEBUG SystemMonitor:2200445 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:40:40,880 DEBUG SystemMonitor:2200445 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:40:40,930 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:40:40,930 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:40:40,930 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:40:40,931 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:40:40,931 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:40:40,931 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,931 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:40:40,931 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:40:40,932 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,932 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:40:41,405 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:43,406 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml +2025-01-07 17:40:43,406 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:47,129 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:47,129 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:47,129 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:40:47,130 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,130 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:40:47,130 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:40:47,130 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:47,131 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:47,131 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:47,131 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:47,131 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:47,132 DEBUG SenderThread:2200445 [sender.py:send():379] send: summary +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:47,132 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:40:47,133 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:47,133 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:40:47,133 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,133 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:40:47,133 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:40:47,133 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:47,133 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:47,133 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:40:47,133 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,133 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:40:47,408 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:47,879 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:50,205 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:40:50,205 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:50,206 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,206 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:40:50,206 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,206 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:40:50,206 INFO SenderThread:2200445 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:40:50,208 INFO SenderThread:2200445 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:40:50,210 INFO SenderThread:2200445 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:40:50,224 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:40:50,225 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,225 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:40:50,225 DEBUG SenderThread:2200445 [sender.py:send():379] send: artifact +2025-01-07 17:40:50,406 INFO SenderThread:2200445 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:40:50,406 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,406 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:40:50,406 INFO SenderThread:2200445 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml config.yaml +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json wandb-summary.json +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt requirements.txt +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:50,410 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log output.log +2025-01-07 17:40:50,410 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch diff.patch +2025-01-07 17:40:50,413 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:40:50,415 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:50,415 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:50,415 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:40:50,415 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,415 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:40:50,416 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,416 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:40:50,416 INFO SenderThread:2200445 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:40:50,607 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml +2025-01-07 17:40:50,608 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:50,618 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt +2025-01-07 17:40:50,648 INFO wandb-upload_4:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:50,717 INFO wandb-upload_3:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/7ml17wwx-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:50,880 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:50,880 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:50,917 INFO Thread-11 (_thread_body):2200445 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:40:50,917 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,917 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:40:50,918 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,918 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:40:50,918 INFO SenderThread:2200445 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:40:50,919 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:40:50,919 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,919 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:40:50,919 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,919 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:40:50,919 INFO SenderThread:2200445 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:40:51,371 INFO SenderThread:2200445 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:40:51,371 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:40:51,371 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:51,371 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:40:51,371 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:51,372 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:40:51,372 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:40:51,372 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:51,372 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:40:51,372 DEBUG SenderThread:2200445 [sender.py:send():379] send: final +2025-01-07 17:40:51,372 DEBUG SenderThread:2200445 [sender.py:send():379] send: footer +2025-01-07 17:40:51,372 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:51,372 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:40:51,373 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:51,373 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:51,373 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:40:51,373 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:51,374 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:40:51,374 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:40:51,374 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:40:51,375 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:51,441 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:51,442 INFO MainThread:2200445 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:40:51,442 INFO MainThread:2200445 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:40:51,442 INFO MainThread:2200445 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:40:51,443 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:40:51,443 INFO HandlerThread:2200445 [handler.py:finish():882] shutting down handler +2025-01-07 17:40:52,374 INFO WriterThread:2200445 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb +2025-01-07 17:40:52,442 INFO SenderThread:2200445 [sender.py:finish():1615] shutting down sender +2025-01-07 17:40:52,442 INFO SenderThread:2200445 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:40:52,442 INFO SenderThread:2200445 [file_pusher.py:join():175] waiting for file pusher