diff --git a/.gitattributes b/.gitattributes
index ec46aec7e1f70c976413a591e55bafb5ee7f5d5a..7cb15be5b0ef81278826c1239f3fb7a176e806b3 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -165,3 +165,33 @@ ppo9w43r/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
 ppo9w43r/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
 ppo9w43r/episode_rosbags/episode_1_2025-01-03-23-11-33.bag filter=lfs diff=lfs merge=lfs -text
 br14p9rb/episode_rosbags/episode_4_2025-01-04-22-17-01.bag filter=lfs diff=lfs merge=lfs -text
+br14p9rb/episode_rosbags/episode_3_2025-01-04-22-15-51.bag filter=lfs diff=lfs merge=lfs -text
+kw7fpvwf/episode_rosbags/episode_2_2025-01-04-20-28-51.bag filter=lfs diff=lfs merge=lfs -text
+kw7fpvwf/episode_rosbags/episode_1_2025-01-04-20-27-50.bag filter=lfs diff=lfs merge=lfs -text
+br14p9rb/episode_rosbags/episode_1_2025-01-04-22-13-27.bag filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+kw7fpvwf/episode_rosbags/episode_3_2025-01-04-20-29-48.bag filter=lfs diff=lfs merge=lfs -text
+kw7fpvwf/episode_rosbags/episode_4_2025-01-04-20-30-37.bag filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/episode_rosbags/episode_0_2025-01-04-22-07-05.bag filter=lfs diff=lfs merge=lfs -text
+kw7fpvwf/episode_rosbags/episode_0_2025-01-04-20-27-02.bag filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/episode_rosbags/episode_2_2025-01-04-22-08-51.bag filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yxgrkg1w/episode_rosbags/episode_3_2025-01-04-22-09-51.bag filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zsx5pj5g/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
diff --git a/br14p9rb/episode_rosbags/episode_1_2025-01-04-22-13-27.bag b/br14p9rb/episode_rosbags/episode_1_2025-01-04-22-13-27.bag
new file mode 100644
index 0000000000000000000000000000000000000000..a73cefa0c6cb262426eb5bd133b7a7a8fa171a12
--- /dev/null
+++ b/br14p9rb/episode_rosbags/episode_1_2025-01-04-22-13-27.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63efa2d16a805b4c2eb2b5b0e243d5f8a6c03d26e3d5047f408d11d0f130f6df
+size 2610935661
diff --git a/br14p9rb/episode_rosbags/episode_3_2025-01-04-22-15-51.bag b/br14p9rb/episode_rosbags/episode_3_2025-01-04-22-15-51.bag
new file mode 100644
index 0000000000000000000000000000000000000000..0cb5f2586b863083dba1880d51a1467554c83e23
--- /dev/null
+++ b/br14p9rb/episode_rosbags/episode_3_2025-01-04-22-15-51.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:859cc09e5b9a731af1c298b56addf10ccf70c3c929c169eaa148e2b4b4068448
+size 2601352299
diff --git a/kw7fpvwf/episode_rosbags/episode_0_2025-01-04-20-27-02.bag b/kw7fpvwf/episode_rosbags/episode_0_2025-01-04-20-27-02.bag
new file mode 100644
index 0000000000000000000000000000000000000000..fb5d84c5ca0b6399333420dce89d6f44358101e7
--- /dev/null
+++ b/kw7fpvwf/episode_rosbags/episode_0_2025-01-04-20-27-02.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4971197b81eb36e47ee624f4d7a3b3fac4af5a5de9b44829cfc42e1743a016
+size 2620171121
diff --git a/kw7fpvwf/episode_rosbags/episode_1_2025-01-04-20-27-50.bag b/kw7fpvwf/episode_rosbags/episode_1_2025-01-04-20-27-50.bag
new file mode 100644
index 0000000000000000000000000000000000000000..ce32474fc30109366032f6903a470797161dd17a
--- /dev/null
+++ b/kw7fpvwf/episode_rosbags/episode_1_2025-01-04-20-27-50.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e86ba1f3c8df33d0b0214afecad169ef4847d1ed6c3752eb5b569a0010c145e6
+size 2604072076
diff --git a/kw7fpvwf/episode_rosbags/episode_2_2025-01-04-20-28-51.bag b/kw7fpvwf/episode_rosbags/episode_2_2025-01-04-20-28-51.bag
new file mode 100644
index 0000000000000000000000000000000000000000..eabc206b05806cf57055489aa6d61841f7818734
--- /dev/null
+++ b/kw7fpvwf/episode_rosbags/episode_2_2025-01-04-20-28-51.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:763eca28d4f8cae8d7eaf51efe8ec7f29d78aa79ac26145e2998a6687d2d9691
+size 2601104748
diff --git a/kw7fpvwf/episode_rosbags/episode_3_2025-01-04-20-29-48.bag b/kw7fpvwf/episode_rosbags/episode_3_2025-01-04-20-29-48.bag
new file mode 100644
index 0000000000000000000000000000000000000000..ef0d1185058e9d29620889df1dc0441c43920477
--- /dev/null
+++ b/kw7fpvwf/episode_rosbags/episode_3_2025-01-04-20-29-48.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66370f11340aae1cb0a4a0488ade52ea387e9d9622672045e4caf75b3a30f425
+size 2619012091
diff --git a/kw7fpvwf/episode_rosbags/episode_4_2025-01-04-20-30-37.bag b/kw7fpvwf/episode_rosbags/episode_4_2025-01-04-20-30-37.bag
new file mode 100644
index 0000000000000000000000000000000000000000..2523f3f3debf39291fb11410aeb2cab8521bbd1a
--- /dev/null
+++ b/kw7fpvwf/episode_rosbags/episode_4_2025-01-04-20-30-37.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3277d3578da8ab0d049034f94d6bac9227a3b3b5b8338e16c7943d921d7c31fd
+size 2607152854
diff --git a/yxgrkg1w/.hydra/config.yaml b/yxgrkg1w/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584
--- /dev/null
+++ b/yxgrkg1w/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1930_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/yxgrkg1w/.hydra/hydra.yaml b/yxgrkg1w/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fc68c700fbf73021c3f9deb1bfc3280d0b0c5442
--- /dev/null
+++ b/yxgrkg1w/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/yxgrkg1w/.hydra/overrides.yaml b/yxgrkg1w/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/yxgrkg1w/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/yxgrkg1w/episode_rosbags/aligned_depth_to_color_K.npy b/yxgrkg1w/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/yxgrkg1w/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/yxgrkg1w/episode_rosbags/color_K.npy b/yxgrkg1w/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/yxgrkg1w/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/yxgrkg1w/episode_rosbags/episode_0_2025-01-04-22-07-05.bag b/yxgrkg1w/episode_rosbags/episode_0_2025-01-04-22-07-05.bag
new file mode 100644
index 0000000000000000000000000000000000000000..e0ce6a9c0ed6f52722dca80a13dae56fb1ab9633
--- /dev/null
+++ b/yxgrkg1w/episode_rosbags/episode_0_2025-01-04-22-07-05.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a38bb4597fdab2be767c47dd5f5c2b033ca60286e483b4df32074e80ef47b2f
+size 2608253040
diff --git a/yxgrkg1w/episode_rosbags/episode_2_2025-01-04-22-08-51.bag b/yxgrkg1w/episode_rosbags/episode_2_2025-01-04-22-08-51.bag
new file mode 100644
index 0000000000000000000000000000000000000000..035c7a4fefec045c73802f5cc3843efcfeb7b3f9
--- /dev/null
+++ b/yxgrkg1w/episode_rosbags/episode_2_2025-01-04-22-08-51.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:348e02a42f885a5f7250c4ed9161c7c3ba84bacbe42580e277c9bdb76918433f
+size 2360309609
diff --git a/yxgrkg1w/episode_rosbags/episode_3_2025-01-04-22-09-51.bag b/yxgrkg1w/episode_rosbags/episode_3_2025-01-04-22-09-51.bag
new file mode 100644
index 0000000000000000000000000000000000000000..cc6c3cdf1e6f7477c0e2b708aa6120227f02ddae
--- /dev/null
+++ b/yxgrkg1w/episode_rosbags/episode_3_2025-01-04-22-09-51.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a919053daeb6da496dff059f3b2008ae1718e0b3d0333e2cbabcc66be5fd04a
+size 2600380294
diff --git a/yxgrkg1w/eval_video/0_eval.mp4 b/yxgrkg1w/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..daf8bffb0a43c4e0ca0ffd0f75b56dc83dd1dc5d
--- /dev/null
+++ b/yxgrkg1w/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2082707a114135740e1a143146867c1fd4ace7ef8a0d66fa77b98c45ba4cf4f0
+size 1524396
diff --git a/yxgrkg1w/eval_video/1_eval.mp4 b/yxgrkg1w/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..03df2972789a27f4d323f0f76766a3074a83c7a7
--- /dev/null
+++ b/yxgrkg1w/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d09445e7e853b8ab7f11b9181a2b6b7e4b1e34a86fdc974989bebcf47dcdb65
+size 1567873
diff --git a/yxgrkg1w/eval_video/2_eval.mp4 b/yxgrkg1w/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bb759b8b804f25ce414ea2c5734091bd02d25bad
--- /dev/null
+++ b/yxgrkg1w/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec34b378e6c06bb4c1326ac17b3313970c7eac8bf66c5fb416b821e45a3e3adb
+size 1434771
diff --git a/yxgrkg1w/eval_video/3_eval.mp4 b/yxgrkg1w/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f126029684ce57675490448d2b5a7924e4187db2
--- /dev/null
+++ b/yxgrkg1w/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abc1529919f51920c0a805d6684f8a06f46f2fc578cf990669520cc68afc18ab
+size 1456646
diff --git a/yxgrkg1w/eval_video/4_eval.mp4 b/yxgrkg1w/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..60ca8c90aa00a680d4bb698af0373c2018ca1aef
--- /dev/null
+++ b/yxgrkg1w/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8162cfa8df033f13066e6b0117a099c20937e2b3c9aa1782a058b78bb6d5e024
+size 1529853
diff --git a/yxgrkg1w/tb/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0 b/yxgrkg1w/tb/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
new file mode 100644
index 0000000000000000000000000000000000000000..a8020efecb53f7b27163d454964ea3102970f1b8
--- /dev/null
+++ b/yxgrkg1w/tb/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3a5bf13af5d07f326ced5e39e3e0ef5c2666516d6a20bf5081626b2262c9e4f
+size 6539
diff --git a/yxgrkg1w/wandb/debug-internal.log b/yxgrkg1w/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..894e719063558e41d0a8884b7dfc62e7c040340f
--- /dev/null
+++ b/yxgrkg1w/wandb/debug-internal.log
@@ -0,0 +1,813 @@
+2025-01-04 22:06:37,267 INFO    StreamThr :2032872 [internal.py:wandb_internal():85] W&B internal server running at pid: 2032872, started at: 2025-01-04 22:06:37.267412
+2025-01-04 22:06:37,269 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status
+2025-01-04 22:06:37,281 INFO    WriterThread:2032872 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb
+2025-01-04 22:06:37,281 DEBUG   SenderThread:2032872 [sender.py:send():379] send: header
+2025-01-04 22:06:37,289 DEBUG   SenderThread:2032872 [sender.py:send():379] send: run
+2025-01-04 22:06:37,522 INFO    SenderThread:2032872 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files
+2025-01-04 22:06:37,522 INFO    SenderThread:2032872 [sender.py:_start_run_threads():1188] run started: yxgrkg1w with start time 1736046397.268647
+2025-01-04 22:06:37,543 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: check_version
+2025-01-04 22:06:37,543 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: check_version
+2025-01-04 22:06:37,586 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: run_start
+2025-01-04 22:06:37,591 DEBUG   HandlerThread:2032872 [system_info.py:__init__():26] System info init
+2025-01-04 22:06:37,591 DEBUG   HandlerThread:2032872 [system_info.py:__init__():41] System info init done
+2025-01-04 22:06:37,591 INFO    HandlerThread:2032872 [system_monitor.py:start():194] Starting system monitor
+2025-01-04 22:06:37,591 INFO    SystemMonitor:2032872 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-04 22:06:37,592 INFO    HandlerThread:2032872 [system_monitor.py:probe():214] Collecting system info
+2025-01-04 22:06:37,592 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started cpu monitoring
+2025-01-04 22:06:37,592 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started disk monitoring
+2025-01-04 22:06:37,593 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started gpu monitoring
+2025-01-04 22:06:37,594 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started memory monitoring
+2025-01-04 22:06:37,594 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started network monitoring
+2025-01-04 22:06:37,605 DEBUG   HandlerThread:2032872 [system_info.py:probe():152] Probing system
+2025-01-04 22:06:37,606 DEBUG   HandlerThread:2032872 [system_info.py:_probe_git():137] Probing git
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_info.py:_probe_git():145] Probing git done
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_info.py:probe():200] Probing system done
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:06:37.605368', 'startedAt': '2025-01-05T03:06:37.253237', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.771874999999, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.125, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.986, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 724.9845581054688}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-04 22:06:37,611 INFO    HandlerThread:2032872 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-04 22:06:37,611 INFO    HandlerThread:2032872 [system_monitor.py:probe():227] Publishing system info
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_info.py:_save_code():44] Saving code
+2025-01-04 22:06:37,613 DEBUG   HandlerThread:2032872 [system_info.py:_save_code():67] Saving code done
+2025-01-04 22:06:37,613 DEBUG   HandlerThread:2032872 [system_info.py:_save_patches():84] Saving git patches
+2025-01-04 22:06:37,675 DEBUG   HandlerThread:2032872 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-04 22:06:37,675 INFO    HandlerThread:2032872 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-04 22:06:37,678 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:06:37,678 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-04 22:06:37,679 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-04 22:06:37,679 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-04 22:06:37,921 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-04 22:06:37,921 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: python_packages
+2025-01-04 22:06:37,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:06:37,922 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:06:37,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:37,924 INFO    HandlerThread:2032872 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/tb"
+  save: true
+}
+control {
+  relay_id: "16536688f07340dfa824309bfebe2328"
+}
+_info {
+  stream_id: "yxgrkg1w"
+}
+
+2025-01-04 22:06:37,925 DEBUG   HandlerThread:2032872 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-04 22:06:37,949 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/wm7r1a6v-wandb-metadata.json
+2025-01-04 22:06:37,951 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/cypcdcrg-diff.patch
+2025-01-04 22:06:37,978 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/iwkr89um-code/FISH/eval_robot.py
+2025-01-04 22:06:38,078 DEBUG   SenderThread:2032872 [sender.py:send():379] send: telemetry
+2025-01-04 22:06:38,078 DEBUG   SenderThread:2032872 [sender.py:send():379] send: tbrecord
+2025-01-04 22:06:38,079 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:06:38,079 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0 with policy live
+2025-01-04 22:06:38,263 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/yep69t78-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code
+2025-01-04 22:06:38,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:39,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:40,523 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:40,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:41,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:42,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:42,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:43,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:44,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:45,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:46,788 DEBUG   SenderThread:2032872 [sender.py:send():379] send: config
+2025-01-04 22:06:46,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:47,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:47,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:48,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:48,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:49,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:50,526 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:50,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:51,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:52,527 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:52,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:06:52,922 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:06:52,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:53,043 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:53,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:54,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:55,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:56,528 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:56,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:57,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:58,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:58,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:59,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:00,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:01,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:02,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:03,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:04,069 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:04,530 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:04,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:05,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:06,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:07,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:07,922 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:07,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:08,533 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:08,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:09,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:10,065 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:10,534 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
+2025-01-04 22:07:10,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:11,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:12,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:13,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:14,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:15,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:15,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:16,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:17,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:18,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:19,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:20,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:20,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:21,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:22,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:22,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:22,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:23,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:24,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:25,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:26,078 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:26,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:27,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:28,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:29,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:30,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:31,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:31,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:32,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:33,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:34,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:35,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:36,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:37,594 DEBUG   SystemMonitor:2032872 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-04 22:07:37,595 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:07:37,595 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:37,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:37,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:37,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:38,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:39,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:40,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:41,659 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:07:41,659 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2082707a114135740e1a.mp4 with policy now
+2025-01-04 22:07:41,659 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:07:41,660 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:07:41,661 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:07:41,661 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:07:41,932 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/q4us9up3-media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:07:41,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:42,542 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media
+2025-01-04 22:07:42,674 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:42,819 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/ckdepwpd-media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:07:42,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:43,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:44,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:44,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:45,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:46,544 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:46,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:47,786 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:47,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:48,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:49,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:50,545 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:50,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:51,546 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:07:51,786 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/1asqqmgl-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:07:51,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:52,546 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:52,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:52,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:52,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:53,085 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:53,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:54,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:55,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:56,547 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:56,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:57,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:58,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:58,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:59,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:00,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:01,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:02,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:03,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:03,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:04,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:05,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:06,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:07,596 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:08:07,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:07,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:07,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:08,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:09,033 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:09,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:10,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:11,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:12,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:13,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:14,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:14,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:15,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:16,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:17,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:18,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:19,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:19,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:20,940 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:21,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:22,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:22,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:22,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:23,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:24,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:25,060 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:25,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:26,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:27,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:28,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:29,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:30,073 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:30,558 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:30,576 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:08:30,620 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:08:30,620 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4 with policy now
+2025-01-04 22:08:30,620 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:08:30,620 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:08:30,631 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:08:30,902 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/284uhvrn-media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:08:30,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:31,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:08:31,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:08:31,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:08:31,843 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/gl7dqx34-media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:08:31,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:32,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:32,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:33,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:33,944 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:34,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:35,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:36,057 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:36,560 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:36,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:37,597 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:08:37,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:37,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:37,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:38,561 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:38,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:39,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:40,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:41,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:42,055 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:42,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:43,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:44,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:45,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:46,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:47,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:47,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:48,360 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:08:48,360 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:08:48,360 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:08:48,565 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:08:48,565 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:08:48,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:49,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:50,565 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:50,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:51,950 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:52,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:52,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:52,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:53,014 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:53,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:54,567 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:54,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:55,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:56,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:57,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:58,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:58,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:59,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:00,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:01,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:02,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:03,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:03,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:04,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:05,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:06,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:07,597 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:09:07,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:07,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:07,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:08,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:09,068 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:09,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:10,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:11,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:12,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:13,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:14,574 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:14,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:14,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:15,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:16,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:17,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:18,575 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:18,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:19,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:19,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:20,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:21,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:22,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:22,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:22,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:23,956 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:24,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:24,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:09:24,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:09:24,580 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:09:24,580 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4 with policy now
+2025-01-04 22:09:24,587 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:09:24,588 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:09:24,588 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:09:24,589 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:09:24,859 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/qbfz1qbr-media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:09:24,955 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:25,127 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/g33ld03i-media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:09:25,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:09:25,631 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:25,958 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:26,578 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:26,967 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:27,578 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:27,962 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:28,967 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:29,966 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:30,579 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:30,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:30,981 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:31,961 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:32,580 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:32,964 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:33,963 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:34,963 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:35,963 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:36,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:36,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:37,598 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:09:37,940 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:37,940 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:37,985 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:38,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:39,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:40,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:41,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:42,092 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:42,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:43,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:44,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:45,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:46,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:47,425 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:47,586 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:09:47,831 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/tz0r8wbr-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:09:47,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:48,208 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:09:48,208 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:09:48,209 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:09:48,587 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:48,587 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:09:48,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:49,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:50,587 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:50,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:51,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:52,455 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:52,588 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:52,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:52,929 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:52,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:53,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:54,589 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:54,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:55,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:56,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:57,893 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:57,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:58,590 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:58,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:59,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:00,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:01,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:02,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:02,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:03,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:04,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:05,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:06,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:07,599 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:10:07,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:07,929 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:07,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:08,086 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:08,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:09,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:10,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:11,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:12,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:13,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:13,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:14,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:15,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:16,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:17,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:18,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:18,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:19,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:20,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:21,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:22,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:22,930 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:22,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:23,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:24,004 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:24,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:25,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:26,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:27,983 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:28,303 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:10:28,303 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4 with policy now
+2025-01-04 22:10:28,314 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:10:28,315 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:10:28,315 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:10:28,315 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:10:28,561 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/0l8v5uyx-media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:10:28,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:28,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:10:28,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:10:28,602 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:10:28,875 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/ih9fynol-media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:10:28,977 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:29,367 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:29,980 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:30,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:30,993 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:31,602 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:31,982 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:32,602 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:32,997 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:33,982 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:34,982 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:35,309 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:35,986 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:36,603 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:36,604 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:10:36,986 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:37,269 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:10:37,269 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:10:37,269 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:10:37,600 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:10:37,604 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:10:37,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:37,934 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:37,986 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:38,604 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:38,996 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:39,993 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:40,382 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:40,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:41,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:42,605 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:42,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:43,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:44,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:45,940 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:45,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:46,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:47,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:48,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:49,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:50,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:50,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:51,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:52,609 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:52,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:52,931 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:52,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:53,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:54,610 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:54,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:55,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:56,365 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:56,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:57,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:58,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:59,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:00,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:01,941 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:01,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:02,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:03,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:04,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:05,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:06,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:06,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:07,600 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:11:07,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:11:07,931 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:11:07,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:08,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:09,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:10,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:11,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:12,084 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:12,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:13,992 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:14,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:15,995 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:16,610 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:11:16,610 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4 with policy now
+2025-01-04 22:11:16,615 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:11:16,616 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:16,616 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:16,619 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:16,619 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:16,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:16,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:11:16,627 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:16,877 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/ku6nm9mw-media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:16,992 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:17,140 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/us00ltlb-media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:17,668 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:17,998 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:18,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:18,998 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:19,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:20,017 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:21,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:22,001 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:22,621 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:22,872 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:22,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:11:22,933 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:11:23,004 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:24,019 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:24,622 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:25,008 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:25,622 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:25,860 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/2rfxiyh8-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:26,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:26,326 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:26,327 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:26,327 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:26,443 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 22:11:26,444 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 22:11:26,586 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media
+2025-01-04 22:11:26,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval
+2025-01-04 22:11:26,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table
+2025-01-04 22:11:26,819 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/n0jbyo5k-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:26,947 INFO    wandb-upload_1:2032872 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnn9viluy
+2025-01-04 22:11:27,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:27,057 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 22:11:27,097 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:11:27,347 INFO    SenderThread:2032872 [sender.py:send_request_log_artifact():1525] logged artifact run-yxgrkg1w-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4MTE3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTAyOA==', 'latestArtifact': None}}
+2025-01-04 22:11:27,347 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:11:27,347 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now
+2025-01-04 22:11:27,347 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:27,347 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,348 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,348 DEBUG   SenderThread:2032872 [sender.py:send():379] send: telemetry
+2025-01-04 22:11:27,348 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,352 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,352 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,352 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,352 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,353 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,353 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,353 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,353 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,354 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,355 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,356 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,356 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,357 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,357 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,358 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,358 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,359 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,359 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,360 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,360 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,361 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,361 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 22:11:27,596 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/pnze7juq-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:27,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:27,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:27,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:27,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval
+2025-01-04 22:11:27,743 INFO    wandb-upload_1:2032872 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4uicmb18
+2025-01-04 22:11:27,845 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/6hfjrk0a-media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:28,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:28,094 INFO    SenderThread:2032872 [sender.py:send_request_log_artifact():1525] logged artifact run-yxgrkg1w-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4MTE5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTAzMg==', 'latestArtifact': None}}
+2025-01-04 22:11:28,099 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:28,240 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:11:28,240 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json with policy now
+2025-01-04 22:11:28,240 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:28,240 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:28,241 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:28,487 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/sxfekju8-media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:28,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:28,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
+2025-01-04 22:11:28,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:29,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:30,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:31,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:32,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:33,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:33,944 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:34,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:35,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:35,370 DEBUG   SenderThread:2032872 [sender.py:send():379] send: exit
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:send_exit():586] handling exit code: 0
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:send_exit():588] handling runtime: 297
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:send_exit():594] send defer
+2025-01-04 22:11:35,370 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,370 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-04 22:11:35,371 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 1
+2025-01-04 22:11:35,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,371 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-04 22:11:35,371 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 2
+2025-01-04 22:11:35,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,371 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-04 22:11:35,371 INFO    HandlerThread:2032872 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-04 22:11:35,371 DEBUG   SystemMonitor:2032872 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-04 22:11:35,371 DEBUG   SystemMonitor:2032872 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-04 22:11:35,372 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-04 22:11:35,372 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined disk monitor
+2025-01-04 22:11:35,425 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-04 22:11:35,425 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined memory monitor
+2025-01-04 22:11:35,426 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined network monitor
+2025-01-04 22:11:35,426 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,426 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-04 22:11:35,426 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 3
+2025-01-04 22:11:35,426 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:11:35,426 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,426 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-04 22:11:35,427 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,427 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-04 22:11:35,427 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 4
+2025-01-04 22:11:35,427 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,427 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-04 22:11:35,627 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:36,627 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:42,333 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:42,333 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:42,333 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-04 22:11:42,334 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,335 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-04 22:11:42,335 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 5
+2025-01-04 22:11:42,335 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:42,335 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:42,335 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,335 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-04 22:11:42,335 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:42,336 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:42,336 DEBUG   SenderThread:2032872 [sender.py:send():379] send: summary
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:42,337 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 6
+2025-01-04 22:11:42,337 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,337 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-04 22:11:42,337 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 7
+2025-01-04 22:11:42,337 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:42,337 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,337 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-04 22:11:42,338 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,338 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-04 22:11:42,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:42,629 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:42,807 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 8
+2025-01-04 22:11:42,807 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:42,807 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,807 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-04 22:11:42,807 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,807 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-04 22:11:42,807 INFO    SenderThread:2032872 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-04 22:11:42,810 INFO    SenderThread:2032872 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-04 22:11:42,812 INFO    SenderThread:2032872 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-04 22:11:42,826 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 9
+2025-01-04 22:11:42,826 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,827 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-04 22:11:42,827 DEBUG   SenderThread:2032872 [sender.py:send():379] send: artifact
+2025-01-04 22:11:43,023 INFO    SenderThread:2032872 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}}
+2025-01-04 22:11:43,023 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:43,023 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-04 22:11:43,023 INFO    SenderThread:2032872 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-04 22:11:43,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:43,630 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:43,630 INFO    SenderThread:2032872 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files
+2025-01-04 22:11:43,630 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml config.yaml
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json wandb-summary.json
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0 events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt requirements.txt
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json wandb-metadata.json
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log output.log
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch diff.patch
+2025-01-04 22:11:43,634 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4 media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4 media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4 media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4 media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4 media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 10
+2025-01-04 22:11:43,637 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:43,638 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:43,640 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-04 22:11:43,640 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:43,640 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-04 22:11:43,640 INFO    SenderThread:2032872 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 22:11:43,815 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:43,843 INFO    wandb-upload_4:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:43,869 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
+2025-01-04 22:11:43,875 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/xq79w8ox-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:43,876 INFO    wandb-upload_3:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt
+2025-01-04 22:11:44,076 INFO    Thread-11 (_thread_body):2032872 [sender.py:transition_state():614] send defer: 11
+2025-01-04 22:11:44,077 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,077 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-04 22:11:44,077 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,077 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-04 22:11:44,077 INFO    SenderThread:2032872 [file_pusher.py:join():175] waiting for file pusher
+2025-01-04 22:11:44,078 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 12
+2025-01-04 22:11:44,079 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,079 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-04 22:11:44,079 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,079 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-04 22:11:44,079 INFO    SenderThread:2032872 [file_stream.py:finish():601] file stream finish called
+2025-01-04 22:11:44,297 INFO    SenderThread:2032872 [file_stream.py:finish():605] file stream finish is done
+2025-01-04 22:11:44,297 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 13
+2025-01-04 22:11:44,297 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,297 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-04 22:11:44,298 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,298 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-04 22:11:44,298 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 14
+2025-01-04 22:11:44,298 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,298 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-04 22:11:44,298 DEBUG   SenderThread:2032872 [sender.py:send():379] send: final
+2025-01-04 22:11:44,298 DEBUG   SenderThread:2032872 [sender.py:send():379] send: footer
+2025-01-04 22:11:44,299 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,299 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-04 22:11:44,299 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:44,299 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: server_info
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-04 22:11:44,300 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: server_info
+2025-01-04 22:11:44,301 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:44,363 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:44,364 INFO    MainThread:2032872 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-04 22:11:44,364 INFO    MainThread:2032872 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-04 22:11:44,364 INFO    MainThread:2032872 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-04 22:11:44,364 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-04 22:11:44,364 INFO    HandlerThread:2032872 [handler.py:finish():882] shutting down handler
+2025-01-04 22:11:45,301 INFO    WriterThread:2032872 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb
+2025-01-04 22:11:45,364 INFO    SenderThread:2032872 [sender.py:finish():1615] shutting down sender
+2025-01-04 22:11:45,364 INFO    SenderThread:2032872 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 22:11:45,364 INFO    SenderThread:2032872 [file_pusher.py:join():175] waiting for file pusher
diff --git a/yxgrkg1w/wandb/debug.log b/yxgrkg1w/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..abf463847a15898a83c67ebf31089391f7ea5874
--- /dev/null
+++ b/yxgrkg1w/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-04 22:06:37,258 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Configure stats pid to 2032698
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/settings
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug.log
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug-internal.log
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():569] calling init triggers
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w'}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():619] starting backend
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():623] setting up manager
+2025-01-04 22:06:37,266 INFO    MainThread:2032698 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-04 22:06:37,268 INFO    MainThread:2032698 [wandb_init.py:init():631] backend started and connected
+2025-01-04 22:06:37,280 INFO    MainThread:2032698 [wandb_init.py:init():720] updated telemetry
+2025-01-04 22:06:37,286 INFO    MainThread:2032698 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-04 22:06:37,543 INFO    MainThread:2032698 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-04 22:06:37,581 INFO    MainThread:2032698 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-04 22:06:37,581 INFO    MainThread:2032698 [wandb_init.py:init():804] starting run threads in backend
+2025-01-04 22:06:37,922 INFO    MainThread:2032698 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-04 22:06:37,922 INFO    MainThread:2032698 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-04 22:06:37,922 INFO    MainThread:2032698 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-04 22:06:37,923 INFO    MainThread:2032698 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-04 22:06:37,923 INFO    MainThread:2032698 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-04 22:06:37,924 INFO    MainThread:2032698 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/tb, True
+2025-01-04 22:06:46,787 INFO    MainThread:2032698 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-04 22:11:45,365 WARNING MsgRouterThr:2032698 [router.py:message_loop():77] message_loop has been closed
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..988a476a502faf4083bda19e5a051ef295288058
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 1
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1930_0'
+wandb_notes:
+  desc: null
+  value: 1930_0_restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736046397
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..f345052a4098779e6d1835354dda24725cd2d7e6
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch
@@ -0,0 +1,56 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 57a851f..7b7e871 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
++# wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1006_0'
++# wandb_run_id: '1926_1' # seed 1
++wandb_run_id: '1930_0' # dataset shuffle seed 1
++
++
+ 
+ # RGBD+mask+contact(contact+DTC)+act history
+-wandb_run_id: '1745_0'
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 71a5d1c..c7eadce 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -159,8 +159,10 @@ run_id_list = [
+ ]
+ 
+ run_id_list = [
+-    # '1006_0',
+-    '1745_0'
++    # '1922_1',
++    # '1922_2',
++    '1926_1',
++    '1930_0',
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py
+index 6f97e26..285afc8 100644
+--- a/gym-envs/gym_envs/envs/hardware/franka.py
++++ b/gym-envs/gym_envs/envs/hardware/franka.py
+@@ -330,8 +330,8 @@ class Franka:
+ 		# self.yaw = yaw
+ 
+ 		self.move_to_target_hz = 10.0
+-		self.move_to_trans_tolerance = 0.005
+-		self.move_to_rot_tolerance = 1.0
++		self.move_to_trans_tolerance = 0.01
++		self.move_to_rot_tolerance = 2.0
+ 		self.move_to_target_max_velocity = 0.8
+ 		self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz)
+ 		self.move_to_target_timeout = 5.0
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]}
\ No newline at end of file
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..29f47b824e9701a70fd6e80f7e62d612d83f42c8
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[45, 55, 0.8181818181818182, 0.6728886961591174, 0.917207777416711, 0.7041302918979557, 0.9086517541783328, 0.7236575483828502, 0.8893923758681664, 0.7373602725090713, 0.8822664878497914]]}
\ No newline at end of file
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4 b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..daf8bffb0a43c4e0ca0ffd0f75b56dc83dd1dc5d
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2082707a114135740e1a143146867c1fd4ace7ef8a0d66fa77b98c45ba4cf4f0
+size 1524396
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4 b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..03df2972789a27f4d323f0f76766a3074a83c7a7
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d09445e7e853b8ab7f11b9181a2b6b7e4b1e34a86fdc974989bebcf47dcdb65
+size 1567873
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4 b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bb759b8b804f25ce414ea2c5734091bd02d25bad
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec34b378e6c06bb4c1326ac17b3313970c7eac8bf66c5fb416b821e45a3e3adb
+size 1434771
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4 b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f126029684ce57675490448d2b5a7924e4187db2
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abc1529919f51920c0a805d6684f8a06f46f2fc578cf990669520cc68afc18ab
+size 1456646
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4 b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..60ca8c90aa00a680d4bb698af0373c2018ca1aef
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8162cfa8df033f13066e6b0117a099c20937e2b3c9aa1782a058b78bb6d5e024
+size 1529853
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..6e694dae6901d53a685a7d5e77e501debf27cacd
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
@@ -0,0 +1,274 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736046406.788570]: resetting environment
+[INFO] [1736046406.796170]: cleared current plan
+[INFO] [1736046406.796952]: moving to home
+[INFO] [1736046407.799638]: reached home
+[INFO] [1736046407.801010]: reset action history
+[INFO] [1736046410.511619]: environment reset
+Starting episode 0
+[INFO] [1736046410.517535]: resetting environment
+[INFO] [1736046410.524709]: cleared current plan
+[INFO] [1736046410.525743]: moving to home
+[INFO] [1736046411.528072]: reached home
+[INFO] [1736046411.529291]: reset action history
+[INFO] [1736046414.238323]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736046422.019185]: resetting environment
+[INFO] [1736046422.023818]: cleared current plan
+[INFO] [1736046422.024791]: moving to home
+[INFO] [1736046423.026302]: reached home
+[INFO] [1736046423.027577]: reset action history
+[INFO] [1736046425.738879]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/episode_rosbags/episode_0_2025-01-04-22-07-05.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2032698.log
+started roslaunch server http://158.130.50.37:33841/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2033001]
+process[print_text-2]: started with pid [2033002]
+process[rosbag_record-3]: started with pid [2033003]
+started bagging!
+[INFO] [1736046460.692357]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1736046461.434043]: Stored episode 1.
+[INFO] [1736046461.435153]: Saving video...
+[INFO] [1736046461.641714]: Video saved!
+Starting episode 1
+[INFO] [1736046461.660470]: resetting environment
+[INFO] [1736046461.673037]: cleared current plan
+[INFO] [1736046461.673335]: moving to home
+[INFO] [1736046465.075738]: reached home
+[INFO] [1736046465.076244]: reset action history
+[INFO] [1736046467.785547]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736046470.567726]: resetting environment
+[INFO] [1736046470.574695]: cleared current plan
+[INFO] [1736046470.574891]: moving to home
+[INFO] [1736046471.576593]: reached home
+[INFO] [1736046471.577006]: reset action history
+[INFO] [1736046474.284900]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/episode_rosbags/episode_1_2025-01-04-22-07-54.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2032698.log
+started roslaunch server http://158.130.50.37:36341/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2033128]
+process[print_text-5]: started with pid [2033129]
+process[rosbag_record-6]: started with pid [2033153]
+started bagging!
+[INFO] [1736046509.221092]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736046510.027682]: Stored episode 2.
+[INFO] [1736046510.027921]: Saving video...
+[INFO] [1736046510.553749]: Video saved!
+Starting episode 2
+[INFO] [1736046510.582393]: resetting environment
+[INFO] [1736046510.608253]: cleared current plan
+[INFO] [1736046510.608728]: moving to home
+[WARN] [1736046510.571425]: Plan exhausted
+[INFO] [1736046514.012514]: reached home
+[INFO] [1736046514.014173]: reset action history
+[INFO] [1736046516.738277]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736046527.983048]: resetting environment
+[INFO] [1736046527.992971]: cleared current plan
+[INFO] [1736046527.994900]: moving to home
+[INFO] [1736046528.997351]: reached home
+[INFO] [1736046528.998010]: reset action history
+[INFO] [1736046531.757566]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/episode_rosbags/episode_2_2025-01-04-22-08-51.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2032698.log
+started roslaunch server http://158.130.50.37:44187/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2033273]
+process[print_text-8]: started with pid [2033274]
+process[rosbag_record-9]: started with pid [2033298]
+started bagging!
+For topic gripper_width: timestamp difference is 101088195 for nearest: 1736046551751512675 - target: 1736046551650424480 at idx 249
+For topic gripper_width: timestamp difference is 105503157 for nearest: 1736046557218214110 - target: 1736046557112710953 at idx 249
+preempting episode
+[INFO] [1736046563.292253]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit
+[print_text-8] killing on exit
+[INFO] [1736046564.145600]: Stored episode 3.
+[INFO] [1736046564.147792]: Saving video...
+[INFO] [1736046564.542725]: Video saved!
+Starting episode 3
+[INFO] [1736046564.590951]: resetting environment
+[INFO] [1736046564.629306]: cleared current plan
+[INFO] [1736046564.630808]: moving to home
+[WARN] [1736046564.572985]: Plan exhausted
+[WARN] [1736046564.611243]: Plan exhausted
+[INFO] [1736046568.038913]: reached home
+[INFO] [1736046568.046939]: reset action history
+[INFO] [1736046570.937258]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736046587.450572]: resetting environment
+[INFO] [1736046587.478534]: cleared current plan
+[INFO] [1736046587.482522]: moving to home
+[INFO] [1736046588.487746]: reached home
+[INFO] [1736046588.489597]: reset action history
+[INFO] [1736046591.267219]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/episode_rosbags/episode_3_2025-01-04-22-09-51.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2032698.log
+started roslaunch server http://158.130.50.37:42683/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2033437]
+process[print_text-11]: started with pid [2033438]
+process[rosbag_record-12]: started with pid [2033439]
+started bagging!
+For topic gripper_width: timestamp difference is 101711655 for nearest: 1736046593718190575 - target: 1736046593616478920 at idx 33
+For topic gripper_width: timestamp difference is 104194022 for nearest: 1736046596851524449 - target: 1736046596747330427 at idx 69
+[INFO] [1736046627.028670]: Storing episode...
+[rosbag_record-12] killing on exit
+[pub_text-10] killing on exit[print_text-11] killing on exit
+[INFO] [1736046627.750266]: Stored episode 4.
+[INFO] [1736046627.760874]: Saving video...
+[INFO] [1736046628.259349]: Video saved!
+Starting episode 4
+[INFO] [1736046628.322777]: resetting environment
+[INFO] [1736046628.367031]: cleared current plan
+[INFO] [1736046628.367254]: moving to home
+[WARN] [1736046628.361191]: Plan exhausted
+[INFO] [1736046631.470496]: reached home
+[INFO] [1736046631.471816]: reset action history
+[INFO] [1736046634.261829]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736046636.382276]: resetting environment
+[INFO] [1736046636.415250]: cleared current plan
+[INFO] [1736046636.421438]: moving to home
+[INFO] [1736046637.426319]: reached home
+[INFO] [1736046637.427768]: reset action history
+[INFO] [1736046640.209802]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/episode_rosbags/episode_4_2025-01-04-22-10-40.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2032698.log
+started roslaunch server http://158.130.50.37:43481/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2033554]
+process[print_text-14]: started with pid [2033555]
+process[rosbag_record-15]: started with pid [2033556]
+started bagging!
+For topic gripper_width: timestamp difference is 104716147 for nearest: 1736046649918185795 - target: 1736046649813469648 at idx 142
+For topic gripper_width: timestamp difference is 109125243 for nearest: 1736046652018185244 - target: 1736046651909060001 at idx 168
+For topic gripper_width: timestamp difference is 111806954 for nearest: 1736046653318196143 - target: 1736046653206389189 at idx 187
+[INFO] [1736046675.250794]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1736046676.072309]: Stored episode 5.
+[INFO] [1736046676.079413]: Saving video...
+[INFO] [1736046676.564688]: Video saved!
+[INFO] [1736046676.616906]: resetting environment
+[INFO] [1736046676.667152]: cleared current plan
+[INFO] [1736046676.668122]: moving to home
+[WARN] [1736046676.599396]: Plan exhausted
+[WARN] [1736046676.635533]: Plan exhausted
+[INFO] [1736046680.081873]: reached home
+[INFO] [1736046680.083142]: reset action history
+[INFO] [1736046682.871330]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9300ad9e772f85f1262c90b178613721ba30983
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-05T03:06:37.605368",
+    "startedAt": "2025-01-05T03:06:37.253237",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 4124.771874999999,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4498.125,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.986,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 724.9845581054688
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..9b488bdf5fd51909421f5c9c9b17b26084edfd54
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "2082707a114135740e1a143146867c1fd4ace7ef8a0d66fa77b98c45ba4cf4f0", "size": 1524396, "path": "media/videos/eval/0_eval_0_2082707a114135740e1a.mp4"}, "global_step": 652, "_timestamp": 1736046686.9672463, "_runtime": 289.6985993385315, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9d09445e7e853b8ab7f11b9181a2b6b7e4b1e34a86fdc974989bebcf47dcdb65", "size": 1567873, "path": "media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4"}, "eval/num_success": 0.0, "episode": 55.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "ec34b378e6c06bb4c1326ac17b3313970c7eac8bf66c5fb416b821e45a3e3adb", "size": 1434771, "path": "media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "abc1529919f51920c0a805d6684f8a06f46f2fc578cf990669520cc68afc18ab", "size": 1456646, "path": "media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "8162cfa8df033f13066e6b0117a099c20937e2b3c9aa1782a058b78bb6d5e024", "size": 1529853, "path": "media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://z6q2g3mff1b5tj32qsggn21wvc6vfwng7xqnypqu21su36l5d9n5me3x4egqvie2vt305c23saywnn4ljjc7v7peeb2456bce4oit9ofav4s5opql6vow7zpw03tiqn3/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qryn3i1fpqnlfigaiyebyuxy4f0isnwhqjmiig13dqj6hjq3ft5xx1j1r0tlsvswohga4jy5mcfj0ftcm8yxtxkljaf6h468p7sjw7o8oaevhwyr6aqdopy3phpies3w:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 45, "total_num_episodes": 55, "total_success_rate": 0.8181818181818182, "total_success_rate_lb_0.01": 0.6728886961591174, "total_success_rate_ub_0.01": 0.917207777416711, "total_success_rate_lb_0.025": 0.7041302918979557, "total_success_rate_ub_0.025": 0.9086517541783328, "total_success_rate_lb_0.05": 0.7236575483828502, "total_success_rate_ub_0.05": 0.8893923758681664, "total_success_rate_lb_0.1": 0.7373602725090713, "total_success_rate_ub_0.1": 0.8822664878497914, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c39907d18b7d33442581806712a363ca985575e78ed31db47ba06d7b7734208d", "size": 519, "artifact_path": "wandb-client-artifact://1zp0ui72sbhp6u9o78u2tgb2oralfo1pev1hiwbed84dd2ytda76cosjtzloey1lhaat1ao4vlao06eul7s2ppyrg71ubkrc03oofhqlehrd926b36uofxqp7dln4pex/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://c0fionhinnqj4y7f7hn5fsaop19sk1jykc5h97snqbr260vll1x54d1psmop8bshtj6z7lzohvy3u85w203nxfc7y8j1u7fuk3p59wmkejkruvr5f3ys63rnwhhx44g4:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 297}, "eval/total_success_rate": 0.8181818127632141, "eval/total_success_rate_lb_0.01": 0.6728886961936951, "eval/total_success_rate_ub_0.01": 0.9172077775001526, "eval/total_success_rate_lb_0.025": 0.7041302919387817, "eval/total_success_rate_ub_0.025": 0.9086517691612244, "eval/total_success_rate_lb_0.05": 0.7236575484275818, "eval/total_success_rate_ub_0.05": 0.8893923759460449, "eval/total_success_rate_lb_0.1": 0.7373602986335754, "eval/total_success_rate_ub_0.1": 0.8822664618492126}
\ No newline at end of file
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug-internal.log b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..894e719063558e41d0a8884b7dfc62e7c040340f
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug-internal.log
@@ -0,0 +1,813 @@
+2025-01-04 22:06:37,267 INFO    StreamThr :2032872 [internal.py:wandb_internal():85] W&B internal server running at pid: 2032872, started at: 2025-01-04 22:06:37.267412
+2025-01-04 22:06:37,269 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status
+2025-01-04 22:06:37,281 INFO    WriterThread:2032872 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb
+2025-01-04 22:06:37,281 DEBUG   SenderThread:2032872 [sender.py:send():379] send: header
+2025-01-04 22:06:37,289 DEBUG   SenderThread:2032872 [sender.py:send():379] send: run
+2025-01-04 22:06:37,522 INFO    SenderThread:2032872 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files
+2025-01-04 22:06:37,522 INFO    SenderThread:2032872 [sender.py:_start_run_threads():1188] run started: yxgrkg1w with start time 1736046397.268647
+2025-01-04 22:06:37,543 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: check_version
+2025-01-04 22:06:37,543 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: check_version
+2025-01-04 22:06:37,586 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: run_start
+2025-01-04 22:06:37,591 DEBUG   HandlerThread:2032872 [system_info.py:__init__():26] System info init
+2025-01-04 22:06:37,591 DEBUG   HandlerThread:2032872 [system_info.py:__init__():41] System info init done
+2025-01-04 22:06:37,591 INFO    HandlerThread:2032872 [system_monitor.py:start():194] Starting system monitor
+2025-01-04 22:06:37,591 INFO    SystemMonitor:2032872 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-04 22:06:37,592 INFO    HandlerThread:2032872 [system_monitor.py:probe():214] Collecting system info
+2025-01-04 22:06:37,592 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started cpu monitoring
+2025-01-04 22:06:37,592 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started disk monitoring
+2025-01-04 22:06:37,593 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started gpu monitoring
+2025-01-04 22:06:37,594 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started memory monitoring
+2025-01-04 22:06:37,594 INFO    SystemMonitor:2032872 [interfaces.py:start():188] Started network monitoring
+2025-01-04 22:06:37,605 DEBUG   HandlerThread:2032872 [system_info.py:probe():152] Probing system
+2025-01-04 22:06:37,606 DEBUG   HandlerThread:2032872 [system_info.py:_probe_git():137] Probing git
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_info.py:_probe_git():145] Probing git done
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_info.py:probe():200] Probing system done
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:06:37.605368', 'startedAt': '2025-01-05T03:06:37.253237', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.771874999999, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.125, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.986, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 724.9845581054688}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-04 22:06:37,611 INFO    HandlerThread:2032872 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-04 22:06:37,611 INFO    HandlerThread:2032872 [system_monitor.py:probe():227] Publishing system info
+2025-01-04 22:06:37,611 DEBUG   HandlerThread:2032872 [system_info.py:_save_code():44] Saving code
+2025-01-04 22:06:37,613 DEBUG   HandlerThread:2032872 [system_info.py:_save_code():67] Saving code done
+2025-01-04 22:06:37,613 DEBUG   HandlerThread:2032872 [system_info.py:_save_patches():84] Saving git patches
+2025-01-04 22:06:37,675 DEBUG   HandlerThread:2032872 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-04 22:06:37,675 INFO    HandlerThread:2032872 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-04 22:06:37,678 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:06:37,678 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-04 22:06:37,679 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-04 22:06:37,679 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-04 22:06:37,921 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-04 22:06:37,921 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: python_packages
+2025-01-04 22:06:37,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:06:37,922 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:06:37,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:37,924 INFO    HandlerThread:2032872 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/tb"
+  save: true
+}
+control {
+  relay_id: "16536688f07340dfa824309bfebe2328"
+}
+_info {
+  stream_id: "yxgrkg1w"
+}
+
+2025-01-04 22:06:37,925 DEBUG   HandlerThread:2032872 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-04 22:06:37,949 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/wm7r1a6v-wandb-metadata.json
+2025-01-04 22:06:37,951 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/cypcdcrg-diff.patch
+2025-01-04 22:06:37,978 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/iwkr89um-code/FISH/eval_robot.py
+2025-01-04 22:06:38,078 DEBUG   SenderThread:2032872 [sender.py:send():379] send: telemetry
+2025-01-04 22:06:38,078 DEBUG   SenderThread:2032872 [sender.py:send():379] send: tbrecord
+2025-01-04 22:06:38,079 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:06:38,079 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0 with policy live
+2025-01-04 22:06:38,263 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/yep69t78-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py
+2025-01-04 22:06:38,524 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH
+2025-01-04 22:06:38,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code
+2025-01-04 22:06:38,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:39,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:40,523 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:40,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:41,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:42,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:42,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:43,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:44,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:45,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:46,788 DEBUG   SenderThread:2032872 [sender.py:send():379] send: config
+2025-01-04 22:06:46,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:47,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:47,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:48,525 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:48,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:49,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:50,526 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:50,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:51,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:52,527 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:52,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:06:52,922 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:06:52,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:53,043 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:53,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:54,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:55,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:56,528 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:06:56,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:57,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:58,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:06:58,925 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:06:59,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:00,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:01,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:02,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:03,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:04,069 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:04,530 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:04,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:05,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:06,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:07,922 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:07,922 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:07,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:08,533 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:08,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:09,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:10,065 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:10,534 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
+2025-01-04 22:07:10,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:11,927 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:12,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:13,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:14,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:15,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:15,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:16,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:17,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:18,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:19,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:20,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:20,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:21,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:22,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:22,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:22,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:23,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:24,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:25,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:26,078 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:26,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:27,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:28,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:29,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:30,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:31,926 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:31,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:32,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:33,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:34,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:35,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:36,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:37,594 DEBUG   SystemMonitor:2032872 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-04 22:07:37,595 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:07:37,595 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:37,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:37,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:37,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:38,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:39,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:40,933 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:41,659 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:07:41,659 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2082707a114135740e1a.mp4 with policy now
+2025-01-04 22:07:41,659 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:07:41,660 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:07:41,661 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:07:41,661 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:07:41,932 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/q4us9up3-media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:07:41,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:42,542 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos
+2025-01-04 22:07:42,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media
+2025-01-04 22:07:42,674 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:42,819 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/ckdepwpd-media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:07:42,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:43,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:44,543 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:44,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:45,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:46,544 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:46,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:47,786 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:47,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:48,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:49,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:50,545 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:50,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:51,546 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:07:51,786 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/1asqqmgl-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:07:51,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:52,546 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:52,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:07:52,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:07:52,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:53,085 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:53,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:54,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:55,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:56,547 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:07:56,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:57,935 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:58,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:07:58,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:07:59,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:00,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:01,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:02,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:03,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:03,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:04,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:05,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:06,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:07,596 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:08:07,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:07,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:07,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:08,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:09,033 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:09,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:10,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:11,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:12,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:13,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:14,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:14,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:15,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:16,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:17,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:18,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:19,928 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:19,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:20,940 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:21,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:22,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:22,923 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:22,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:23,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:24,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:25,060 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:25,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:26,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:27,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:28,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:29,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:30,073 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:30,558 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:30,576 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:08:30,620 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:08:30,620 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4 with policy now
+2025-01-04 22:08:30,620 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:08:30,620 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:08:30,631 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:08:30,902 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/284uhvrn-media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:08:30,943 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:31,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:08:31,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:08:31,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:08:31,843 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/gl7dqx34-media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:08:31,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:32,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:32,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:33,559 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:33,944 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:34,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:35,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:36,057 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:36,560 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:36,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:37,597 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:08:37,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:37,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:37,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:38,561 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:38,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:39,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:40,945 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:41,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:42,055 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:42,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:43,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:44,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:45,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:46,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:47,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:47,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:48,360 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:08:48,360 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:08:48,360 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:08:48,565 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:08:48,565 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:08:48,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:49,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:50,565 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:50,946 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:51,950 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:52,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:08:52,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:08:52,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:53,014 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:53,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:54,567 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:08:54,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:55,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:56,947 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:57,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:58,930 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:08:58,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:08:59,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:00,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:01,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:02,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:03,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:03,948 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:04,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:05,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:06,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:07,597 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:09:07,923 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:07,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:07,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:08,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:09,068 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:09,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:10,949 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:11,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:12,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:13,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:14,574 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:14,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:14,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:15,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:16,952 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:17,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:18,575 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:18,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:19,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:19,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:20,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:21,953 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:22,924 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:22,924 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:22,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:23,956 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:24,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:24,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:09:24,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:09:24,580 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:09:24,580 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4 with policy now
+2025-01-04 22:09:24,587 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:09:24,588 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:09:24,588 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:09:24,589 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:09:24,859 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/qbfz1qbr-media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:09:24,955 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:25,127 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/g33ld03i-media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:09:25,577 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:09:25,631 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:25,958 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:26,578 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:26,967 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:27,578 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:27,962 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:28,967 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:29,966 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:30,579 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:30,937 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:30,981 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:31,961 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:32,580 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:32,964 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:33,963 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:34,963 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:35,963 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:36,936 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:36,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:37,598 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:09:37,940 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:37,940 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:37,985 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:38,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:39,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:40,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:41,965 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:42,092 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:42,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:43,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:44,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:45,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:46,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:47,425 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:47,586 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:09:47,831 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/tz0r8wbr-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:09:47,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:48,208 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:09:48,208 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:09:48,209 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:09:48,587 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:48,587 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:09:48,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:49,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:50,587 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:50,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:51,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:52,455 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:52,588 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:52,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:09:52,929 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:09:52,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:53,970 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:54,589 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:54,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:55,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:56,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:57,893 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:09:57,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:58,590 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:09:58,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:09:59,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:00,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:01,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:02,938 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:02,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:03,971 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:04,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:05,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:06,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:07,599 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:10:07,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:07,929 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:07,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:08,086 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:08,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:09,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:10,972 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:11,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:12,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:13,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:13,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:14,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:15,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:16,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:17,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:18,939 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:18,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:19,973 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:20,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:21,974 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:22,929 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:22,930 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:22,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:23,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:24,004 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:24,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:25,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:26,975 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:27,983 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:28,303 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:10:28,303 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4 with policy now
+2025-01-04 22:10:28,314 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:10:28,315 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:10:28,315 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:10:28,315 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:10:28,561 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/0l8v5uyx-media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:10:28,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:28,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:10:28,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:10:28,602 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:10:28,875 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/ih9fynol-media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:10:28,977 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:29,367 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:29,980 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:30,601 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:30,993 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:31,602 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:31,982 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:32,602 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:32,997 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:33,982 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:34,982 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:35,309 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:35,986 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:36,603 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:36,604 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:10:36,986 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:37,269 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:10:37,269 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:10:37,269 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:10:37,600 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:10:37,604 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:10:37,934 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:37,934 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:37,986 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:38,604 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:38,996 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:39,993 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:40,382 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:40,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:41,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:42,605 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:42,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:43,987 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:44,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:45,940 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:45,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:46,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:47,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:48,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:49,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:50,969 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:50,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:51,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:52,609 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:52,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:10:52,931 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:10:52,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:53,988 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:54,610 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:10:54,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:55,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:56,365 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:10:56,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:57,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:58,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:10:59,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:00,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:01,941 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:01,989 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:02,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:03,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:04,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:05,990 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:06,942 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:06,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:07,600 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:11:07,931 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:11:07,931 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:11:07,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:08,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:09,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:10,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:11,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:12,084 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:12,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:13,992 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:14,991 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:15,995 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:16,610 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:11:16,610 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4 with policy now
+2025-01-04 22:11:16,615 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:11:16,616 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:16,616 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:16,619 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:16,619 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:16,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:16,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval
+2025-01-04 22:11:16,627 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:16,877 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/ku6nm9mw-media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:16,992 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:17,140 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/us00ltlb-media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:17,668 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:17,998 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:18,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:18,998 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:19,620 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:20,017 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:21,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:22,001 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:22,621 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:22,872 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:22,932 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 22:11:22,933 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 22:11:23,004 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:24,019 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:24,622 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:25,008 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:25,622 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:25,860 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/2rfxiyh8-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:26,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:26,326 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:26,327 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:26,327 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:26,443 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 22:11:26,444 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 22:11:26,586 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:26,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media
+2025-01-04 22:11:26,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval
+2025-01-04 22:11:26,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table
+2025-01-04 22:11:26,819 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/n0jbyo5k-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:26,947 INFO    wandb-upload_1:2032872 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnn9viluy
+2025-01-04 22:11:27,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:27,057 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 22:11:27,097 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 22:11:27,347 INFO    SenderThread:2032872 [sender.py:send_request_log_artifact():1525] logged artifact run-yxgrkg1w-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4MTE3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTAyOA==', 'latestArtifact': None}}
+2025-01-04 22:11:27,347 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:11:27,347 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now
+2025-01-04 22:11:27,347 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:27,347 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,348 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,348 DEBUG   SenderThread:2032872 [sender.py:send():379] send: telemetry
+2025-01-04 22:11:27,348 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,352 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,352 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,352 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,352 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,353 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,353 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,353 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,353 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,354 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,355 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,356 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,356 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,357 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,357 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,358 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,358 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,359 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,359 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,360 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,360 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:27,361 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:27,361 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 22:11:27,596 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/pnze7juq-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:27,623 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:27,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:27,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:27,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval
+2025-01-04 22:11:27,743 INFO    wandb-upload_1:2032872 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4uicmb18
+2025-01-04 22:11:27,845 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/6hfjrk0a-media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:28,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:28,094 INFO    SenderThread:2032872 [sender.py:send_request_log_artifact():1525] logged artifact run-yxgrkg1w-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4MTE5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTAzMg==', 'latestArtifact': None}}
+2025-01-04 22:11:28,099 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:28,240 DEBUG   SenderThread:2032872 [sender.py:send():379] send: files
+2025-01-04 22:11:28,240 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json with policy now
+2025-01-04 22:11:28,240 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:28,240 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:28,241 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:28,487 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/sxfekju8-media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:28,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:28,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
+2025-01-04 22:11:28,624 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:29,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:30,005 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:31,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:32,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:33,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:33,944 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:34,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:35,006 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:35,370 DEBUG   SenderThread:2032872 [sender.py:send():379] send: exit
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:send_exit():586] handling exit code: 0
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:send_exit():588] handling runtime: 297
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:35,370 INFO    SenderThread:2032872 [sender.py:send_exit():594] send defer
+2025-01-04 22:11:35,370 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,370 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-04 22:11:35,371 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 1
+2025-01-04 22:11:35,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,371 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-04 22:11:35,371 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-04 22:11:35,371 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 2
+2025-01-04 22:11:35,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,371 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-04 22:11:35,371 INFO    HandlerThread:2032872 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-04 22:11:35,371 DEBUG   SystemMonitor:2032872 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-04 22:11:35,371 DEBUG   SystemMonitor:2032872 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-04 22:11:35,372 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-04 22:11:35,372 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined disk monitor
+2025-01-04 22:11:35,425 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-04 22:11:35,425 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined memory monitor
+2025-01-04 22:11:35,426 INFO    HandlerThread:2032872 [interfaces.py:finish():200] Joined network monitor
+2025-01-04 22:11:35,426 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,426 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-04 22:11:35,426 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 3
+2025-01-04 22:11:35,426 DEBUG   SenderThread:2032872 [sender.py:send():379] send: stats
+2025-01-04 22:11:35,426 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,426 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-04 22:11:35,427 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:35,427 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-04 22:11:35,427 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 4
+2025-01-04 22:11:35,427 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:35,427 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-04 22:11:35,627 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:36,627 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:42,333 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:42,333 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:42,333 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-04 22:11:42,334 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,335 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-04 22:11:42,335 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 5
+2025-01-04 22:11:42,335 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:42,335 DEBUG   SenderThread:2032872 [sender.py:send():379] send: history
+2025-01-04 22:11:42,335 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,335 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-04 22:11:42,335 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 22:11:42,336 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:42,336 DEBUG   SenderThread:2032872 [sender.py:send():379] send: summary
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 22:11:42,337 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 6
+2025-01-04 22:11:42,337 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,337 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-04 22:11:42,337 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-04 22:11:42,337 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 7
+2025-01-04 22:11:42,337 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 22:11:42,337 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,337 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-04 22:11:42,338 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,338 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-04 22:11:42,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:42,629 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:42,807 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 8
+2025-01-04 22:11:42,807 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:42,807 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,807 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-04 22:11:42,807 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:42,807 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-04 22:11:42,807 INFO    SenderThread:2032872 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-04 22:11:42,810 INFO    SenderThread:2032872 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-04 22:11:42,812 INFO    SenderThread:2032872 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-04 22:11:42,826 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 9
+2025-01-04 22:11:42,826 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:42,827 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-04 22:11:42,827 DEBUG   SenderThread:2032872 [sender.py:send():379] send: artifact
+2025-01-04 22:11:43,023 INFO    SenderThread:2032872 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}}
+2025-01-04 22:11:43,023 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:43,023 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-04 22:11:43,023 INFO    SenderThread:2032872 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-04 22:11:43,371 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:43,630 INFO    Thread-12 :2032872 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:43,630 INFO    SenderThread:2032872 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files
+2025-01-04 22:11:43,630 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml config.yaml
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json wandb-summary.json
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0 events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt requirements.txt
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-metadata.json wandb-metadata.json
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log output.log
+2025-01-04 22:11:43,631 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/diff.patch diff.patch
+2025-01-04 22:11:43,634 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4 media/videos/eval/3_eval_3_abc1529919f51920c0a8.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4 media/videos/eval/4_eval_4_8162cfa8df033f13066e.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4 media/videos/eval/2_eval_2_ec34b378e6c06bb4c132.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/0_eval_0_2082707a114135740e1a.mp4 media/videos/eval/0_eval_0_2082707a114135740e1a.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4 media/videos/eval/1_eval_1_9d09445e7e853b8ab7f1.mp4
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json media/table/eval/total_success_rate_ci_6_c39907d18b7d33442581.table.json
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json
+2025-01-04 22:11:43,635 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 10
+2025-01-04 22:11:43,637 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:43,638 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:43,640 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-04 22:11:43,640 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:43,640 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-04 22:11:43,640 INFO    SenderThread:2032872 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 22:11:43,815 INFO    wandb-upload_1:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/wandb-summary.json
+2025-01-04 22:11:43,843 INFO    wandb-upload_4:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/output.log
+2025-01-04 22:11:43,869 INFO    wandb-upload_0:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/config.yaml
+2025-01-04 22:11:43,875 INFO    wandb-upload_2:2032872 [upload_job.py:push():130] Uploaded file /tmp/tmptbheud9awandb/xq79w8ox-events.out.tfevents.1736046397.leonmkim-ROG-Strix-G15CS-G15CS.2032698.0
+2025-01-04 22:11:43,876 INFO    wandb-upload_3:2032872 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/files/requirements.txt
+2025-01-04 22:11:44,076 INFO    Thread-11 (_thread_body):2032872 [sender.py:transition_state():614] send defer: 11
+2025-01-04 22:11:44,077 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,077 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-04 22:11:44,077 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,077 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-04 22:11:44,077 INFO    SenderThread:2032872 [file_pusher.py:join():175] waiting for file pusher
+2025-01-04 22:11:44,078 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 12
+2025-01-04 22:11:44,079 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,079 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-04 22:11:44,079 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,079 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-04 22:11:44,079 INFO    SenderThread:2032872 [file_stream.py:finish():601] file stream finish called
+2025-01-04 22:11:44,297 INFO    SenderThread:2032872 [file_stream.py:finish():605] file stream finish is done
+2025-01-04 22:11:44,297 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 13
+2025-01-04 22:11:44,297 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,297 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-04 22:11:44,298 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,298 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-04 22:11:44,298 INFO    SenderThread:2032872 [sender.py:transition_state():614] send defer: 14
+2025-01-04 22:11:44,298 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 22:11:44,298 INFO    HandlerThread:2032872 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-04 22:11:44,298 DEBUG   SenderThread:2032872 [sender.py:send():379] send: final
+2025-01-04 22:11:44,298 DEBUG   SenderThread:2032872 [sender.py:send():379] send: footer
+2025-01-04 22:11:44,299 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: defer
+2025-01-04 22:11:44,299 INFO    SenderThread:2032872 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-04 22:11:44,299 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:44,299 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: server_info
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-04 22:11:44,300 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-04 22:11:44,300 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: server_info
+2025-01-04 22:11:44,301 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 22:11:44,363 DEBUG   SenderThread:2032872 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 22:11:44,364 INFO    MainThread:2032872 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-04 22:11:44,364 INFO    MainThread:2032872 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-04 22:11:44,364 INFO    MainThread:2032872 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-04 22:11:44,364 DEBUG   HandlerThread:2032872 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-04 22:11:44,364 INFO    HandlerThread:2032872 [handler.py:finish():882] shutting down handler
+2025-01-04 22:11:45,301 INFO    WriterThread:2032872 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb
+2025-01-04 22:11:45,364 INFO    SenderThread:2032872 [sender.py:finish():1615] shutting down sender
+2025-01-04 22:11:45,364 INFO    SenderThread:2032872 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 22:11:45,364 INFO    SenderThread:2032872 [file_pusher.py:join():175] waiting for file pusher
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug.log b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..abf463847a15898a83c67ebf31089391f7ea5874
--- /dev/null
+++ b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-04 22:06:37,258 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Configure stats pid to 2032698
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/settings
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug.log
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/logs/debug-internal.log
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():569] calling init triggers
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w'}
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():619] starting backend
+2025-01-04 22:06:37,259 INFO    MainThread:2032698 [wandb_init.py:init():623] setting up manager
+2025-01-04 22:06:37,266 INFO    MainThread:2032698 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-04 22:06:37,268 INFO    MainThread:2032698 [wandb_init.py:init():631] backend started and connected
+2025-01-04 22:06:37,280 INFO    MainThread:2032698 [wandb_init.py:init():720] updated telemetry
+2025-01-04 22:06:37,286 INFO    MainThread:2032698 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-04 22:06:37,543 INFO    MainThread:2032698 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-04 22:06:37,581 INFO    MainThread:2032698 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-04 22:06:37,581 INFO    MainThread:2032698 [wandb_init.py:init():804] starting run threads in backend
+2025-01-04 22:06:37,922 INFO    MainThread:2032698 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-04 22:06:37,922 INFO    MainThread:2032698 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-04 22:06:37,922 INFO    MainThread:2032698 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-04 22:06:37,923 INFO    MainThread:2032698 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-04 22:06:37,923 INFO    MainThread:2032698 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-04 22:06:37,924 INFO    MainThread:2032698 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/yxgrkg1w/tb, True
+2025-01-04 22:06:46,787 INFO    MainThread:2032698 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-04 22:11:45,365 WARNING MsgRouterThr:2032698 [router.py:message_loop():77] message_loop has been closed
diff --git a/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..db2cd254b0bece1accc35675c128eeb12a19e1e3
Binary files /dev/null and b/yxgrkg1w/wandb/run-20250104_220637-yxgrkg1w/run-yxgrkg1w.wandb differ
diff --git a/zsx5pj5g/episode_rosbags/aligned_depth_to_color_K.npy b/zsx5pj5g/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/zsx5pj5g/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/zsx5pj5g/episode_rosbags/cam_tf_world.npy b/zsx5pj5g/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/zsx5pj5g/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/zsx5pj5g/episode_rosbags/color_K.npy b/zsx5pj5g/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/zsx5pj5g/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/zsx5pj5g/episode_rosbags/depth_K.npy b/zsx5pj5g/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/zsx5pj5g/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/zsx5pj5g/eval_robot.log b/zsx5pj5g/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..a53a0ef6d62d0807dd5b86dff00f2bf7abe6e3bf
--- /dev/null
+++ b/zsx5pj5g/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-04 20:31:52,583][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-04 20:31:52,586][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/zsx5pj5g/eval_video/0_eval.mp4 b/zsx5pj5g/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..63abe916f8e9af4b906f0a76358f991eccffb437
--- /dev/null
+++ b/zsx5pj5g/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f959e9c50cf3c3a02940b1c093c5f31d1ee65c40cebd70655e74088c0110fc
+size 1499363
diff --git a/zsx5pj5g/eval_video/1_eval.mp4 b/zsx5pj5g/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7c98c27f9ae8ff7042954be0529e3033342950e0
--- /dev/null
+++ b/zsx5pj5g/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b2a0a5a222317bf8a15858c4651a4b33f187b761ddbe09d3cf9d26d948217a7
+size 1494961
diff --git a/zsx5pj5g/eval_video/2_eval.mp4 b/zsx5pj5g/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f12a700a0b428210999d362eaf5d47a20b99f9f9
--- /dev/null
+++ b/zsx5pj5g/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13d165b6bcf603a161c53a6adcc616c31483bd32f7b03d6d459b80355c0ea8c1
+size 1496842
diff --git a/zsx5pj5g/eval_video/3_eval.mp4 b/zsx5pj5g/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..00889217ac3f1a52f02e35dd8d949e300caa6125
--- /dev/null
+++ b/zsx5pj5g/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378a37ba58c54e86d1ce679bed3377ae8b329965e85371569be0e53c2a9a51f0
+size 1512946
diff --git a/zsx5pj5g/eval_video/4_eval.mp4 b/zsx5pj5g/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4d79217710a815541d32eb969593990a1ea5daf7
--- /dev/null
+++ b/zsx5pj5g/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a49e5d967d992227eafdc78f00824719e0350cf4dbfe490a10fe06331ff189
+size 1544024
diff --git a/zsx5pj5g/tb/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0 b/zsx5pj5g/tb/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
new file mode 100644
index 0000000000000000000000000000000000000000..3610d6d06126d0b939ad5836c3d77ffe38983e44
--- /dev/null
+++ b/zsx5pj5g/tb/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b3c1d571699c3cfc2cbca861c83fb3509b6edac6d6e6d51ffd1da85183715c3
+size 6539
diff --git a/zsx5pj5g/wandb/debug-internal.log b/zsx5pj5g/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..8010c11fa23c3705c965fcc73fbe7c1063addddc
--- /dev/null
+++ b/zsx5pj5g/wandb/debug-internal.log
@@ -0,0 +1,779 @@
+2025-01-04 20:31:58,357 INFO    StreamThr :2012081 [internal.py:wandb_internal():85] W&B internal server running at pid: 2012081, started at: 2025-01-04 20:31:58.356937
+2025-01-04 20:31:58,358 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status
+2025-01-04 20:31:58,369 INFO    WriterThread:2012081 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb
+2025-01-04 20:31:58,369 DEBUG   SenderThread:2012081 [sender.py:send():379] send: header
+2025-01-04 20:31:58,378 DEBUG   SenderThread:2012081 [sender.py:send():379] send: run
+2025-01-04 20:31:58,612 INFO    SenderThread:2012081 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files
+2025-01-04 20:31:58,612 INFO    SenderThread:2012081 [sender.py:_start_run_threads():1188] run started: zsx5pj5g with start time 1736040718.357367
+2025-01-04 20:31:58,635 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: check_version
+2025-01-04 20:31:58,635 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: check_version
+2025-01-04 20:31:58,680 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: run_start
+2025-01-04 20:31:58,684 DEBUG   HandlerThread:2012081 [system_info.py:__init__():26] System info init
+2025-01-04 20:31:58,685 DEBUG   HandlerThread:2012081 [system_info.py:__init__():41] System info init done
+2025-01-04 20:31:58,685 INFO    HandlerThread:2012081 [system_monitor.py:start():194] Starting system monitor
+2025-01-04 20:31:58,685 INFO    SystemMonitor:2012081 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-04 20:31:58,685 INFO    HandlerThread:2012081 [system_monitor.py:probe():214] Collecting system info
+2025-01-04 20:31:58,686 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started cpu monitoring
+2025-01-04 20:31:58,687 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started disk monitoring
+2025-01-04 20:31:58,688 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started gpu monitoring
+2025-01-04 20:31:58,688 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started memory monitoring
+2025-01-04 20:31:58,688 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started network monitoring
+2025-01-04 20:31:58,698 DEBUG   HandlerThread:2012081 [system_info.py:probe():152] Probing system
+2025-01-04 20:31:58,700 DEBUG   HandlerThread:2012081 [system_info.py:_probe_git():137] Probing git
+2025-01-04 20:31:58,704 DEBUG   HandlerThread:2012081 [system_info.py:_probe_git():145] Probing git done
+2025-01-04 20:31:58,705 DEBUG   HandlerThread:2012081 [system_info.py:probe():200] Probing system done
+2025-01-04 20:31:58,705 DEBUG   HandlerThread:2012081 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:31:58.698894', 'startedAt': '2025-01-05T01:31:58.341612', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.790875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.99, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 594.1672897338867}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-04 20:31:58,705 INFO    HandlerThread:2012081 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-04 20:31:58,705 INFO    HandlerThread:2012081 [system_monitor.py:probe():227] Publishing system info
+2025-01-04 20:31:58,705 DEBUG   HandlerThread:2012081 [system_info.py:_save_code():44] Saving code
+2025-01-04 20:31:58,706 DEBUG   HandlerThread:2012081 [system_info.py:_save_code():67] Saving code done
+2025-01-04 20:31:58,707 DEBUG   HandlerThread:2012081 [system_info.py:_save_patches():84] Saving git patches
+2025-01-04 20:31:58,775 DEBUG   HandlerThread:2012081 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-04 20:31:58,775 INFO    HandlerThread:2012081 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-04 20:31:58,779 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:31:58,779 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-04 20:31:58,779 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-04 20:31:58,780 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-04 20:31:59,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-04 20:31:59,018 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: python_packages
+2025-01-04 20:31:59,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:31:59,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:31:59,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:31:59,021 INFO    HandlerThread:2012081 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/tb"
+  save: true
+}
+control {
+  relay_id: "5eecefe1ab4e42da905d511a8b6149f8"
+}
+_info {
+  stream_id: "zsx5pj5g"
+}
+
+2025-01-04 20:31:59,022 DEBUG   HandlerThread:2012081 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-04 20:31:59,060 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/9fe428ne-wandb-metadata.json
+2025-01-04 20:31:59,078 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/0w0bfaur-diff.patch
+2025-01-04 20:31:59,099 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/s42dfmkq-code/FISH/eval_robot.py
+2025-01-04 20:31:59,126 DEBUG   SenderThread:2012081 [sender.py:send():379] send: telemetry
+2025-01-04 20:31:59,126 DEBUG   SenderThread:2012081 [sender.py:send():379] send: tbrecord
+2025-01-04 20:31:59,127 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:31:59,127 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0 with policy live
+2025-01-04 20:31:59,309 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/91qfc23b-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch
+2025-01-04 20:31:59,615 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH
+2025-01-04 20:31:59,615 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code
+2025-01-04 20:32:00,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:01,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:01,613 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:02,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:03,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:04,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:04,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:05,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:05,295 DEBUG   SenderThread:2012081 [sender.py:send():379] send: config
+2025-01-04 20:32:06,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:07,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:07,615 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:08,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:09,057 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:09,058 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:10,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:11,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:11,616 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:12,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:13,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:14,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:14,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:14,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:14,152 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:15,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:15,617 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:16,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:17,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:18,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:19,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:19,618 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:20,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:20,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:21,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:22,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:23,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:23,619 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:24,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:25,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:25,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:26,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:27,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:28,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:29,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:29,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:29,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:30,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:30,181 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:30,621 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
+2025-01-04 20:32:31,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:32,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:33,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:34,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:35,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:36,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:36,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:37,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:38,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:39,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:40,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:41,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:42,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:42,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:43,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:44,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:44,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:44,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:45,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:46,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:47,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:47,177 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:48,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:49,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:50,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:51,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:52,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:53,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:53,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:54,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:55,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:56,027 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:57,028 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:57,213 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:32:57,214 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:32:57,214 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4 with policy now
+2025-01-04 20:32:57,214 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:32:57,214 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:32:57,215 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:32:57,480 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/i9tx05it-media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:32:57,894 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/okjmxkum-media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:32:58,028 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:58,215 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:58,689 DEBUG   SystemMonitor:2012081 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-04 20:32:58,689 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:32:59,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:59,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:59,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:59,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:00,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:01,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:02,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:03,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:03,610 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:03,631 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:04,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:05,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:05,632 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:06,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:07,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:08,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:09,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:09,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:10,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:11,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:12,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:13,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:14,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:14,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:14,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:14,162 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:15,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:16,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:17,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:18,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:18,636 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:33:18,871 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/lh46lxe0-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:33:19,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:19,232 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:19,636 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:20,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:21,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:21,637 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:22,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:23,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:23,637 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:24,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:25,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:25,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:26,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:27,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:27,639 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:28,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:28,690 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:33:29,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:29,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:29,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:30,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:30,134 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:31,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:32,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:33,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:34,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:35,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:36,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:36,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:37,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:38,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:39,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:39,643 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:40,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:41,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:41,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:41,643 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:42,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:43,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:44,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:44,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:44,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:45,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:46,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:46,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:47,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:48,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:49,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:50,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:51,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:52,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:52,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:53,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:54,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:55,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:56,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:57,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:57,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:58,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:58,422 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:33:58,422 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4 with policy now
+2025-01-04 20:33:58,425 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:33:58,425 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:33:58,426 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:33:58,428 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:33:58,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:33:58,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:33:58,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:33:58,688 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/dkat7zbw-media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:33:58,691 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:33:58,972 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/h513uszj-media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:33:59,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:59,024 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:59,038 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:59,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:00,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:00,651 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:01,037 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:02,046 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:02,102 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:03,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:03,652 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:04,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:05,038 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:06,038 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:06,654 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:34:07,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:07,448 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:34:07,448 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:34:07,448 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:07,448 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:34:07,654 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:07,654 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:34:08,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:09,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:09,655 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:10,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:11,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:11,655 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:12,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:12,565 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:13,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:14,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:14,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:14,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:15,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:16,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:17,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:18,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:18,092 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:19,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:20,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:21,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:22,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:23,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:24,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:24,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:25,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:26,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:27,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:28,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:28,691 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:34:29,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:29,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:29,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:29,076 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:30,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:31,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:32,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:33,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:34,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:35,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:35,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:36,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:37,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:38,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:39,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:40,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:40,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:41,045 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:42,045 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:43,045 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:44,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:44,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:44,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:45,046 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:45,106 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:46,049 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:46,684 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:34:46,685 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4 with policy now
+2025-01-04 20:34:46,687 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:34:46,688 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:34:46,688 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:34:46,695 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:34:46,973 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/m7is78bd-media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:34:47,048 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:47,667 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:47,667 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:34:47,667 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:34:47,668 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:34:47,982 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/s93af7zb-media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:34:48,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:48,668 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:49,050 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:50,055 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:50,427 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:51,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:51,669 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:52,057 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:53,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:54,054 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:54,670 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:34:54,902 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/yyioaer2-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:34:55,070 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:55,284 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:34:55,285 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:34:55,285 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:34:55,659 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:55,670 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:55,670 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:34:56,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:57,070 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:57,671 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:58,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:58,692 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:34:59,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:59,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:59,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:59,672 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:00,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:01,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:01,173 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:01,672 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:02,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:03,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:04,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:05,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:06,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:07,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:07,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:08,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:09,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:10,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:11,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:12,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:13,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:13,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:14,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:14,021 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:14,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:15,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:16,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:17,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:18,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:18,129 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:19,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:20,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:21,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:22,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:23,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:24,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:24,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:25,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:26,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:27,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:28,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:28,692 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:35:29,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:29,021 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:29,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:29,103 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:30,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:31,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:32,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:33,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:34,071 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:34,228 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:34,645 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:35:34,645 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4 with policy now
+2025-01-04 20:35:34,648 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:35:34,649 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:35:34,649 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:35:34,669 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:35:34,683 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:35:34,683 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:35:34,684 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:35:34,936 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/t1f5m9tn-media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:35:35,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:35,204 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/hojre09c-media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:35:35,684 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:36,066 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:37,073 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:38,071 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:39,078 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:39,685 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:40,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:40,072 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:41,073 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:42,075 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:43,087 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:43,686 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:44,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:44,029 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:44,086 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:45,066 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:45,076 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:45,342 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:35:45,342 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:35:45,343 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:35:45,687 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:35:45,687 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:35:46,077 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:47,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:47,688 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:48,088 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:49,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:49,688 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:50,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:50,271 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:51,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:51,689 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:52,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:53,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:54,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:55,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:56,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:56,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:57,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:58,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:58,693 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:35:59,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:59,024 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:59,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:00,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:01,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:01,148 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:02,081 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:03,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:04,081 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:05,082 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:06,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:07,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:07,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:08,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:09,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:10,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:11,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:12,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:12,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:13,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:14,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:36:14,024 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:36:14,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:15,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:16,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:17,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:17,129 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:18,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:19,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:20,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:21,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:22,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:23,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:23,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:24,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:25,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:26,091 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:26,513 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:36:26,513 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4 with policy now
+2025-01-04 20:36:26,514 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:36:26,514 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:26,514 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:26,521 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:26,702 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:26,702 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:26,703 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:36:26,788 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/6xzu7tyn-media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:27,036 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/aybqvku0-media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:27,095 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:27,703 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:28,099 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:28,583 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:28,693 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:36:28,703 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:29,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:36:29,030 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:36:29,098 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:30,098 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:31,101 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:31,704 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:32,101 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:33,145 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:34,048 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:34,103 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:35,110 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:35,705 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:36,108 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:36,370 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 20:36:36,370 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 20:36:36,499 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:36:36,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:36,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:36,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media
+2025-01-04 20:36:36,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval
+2025-01-04 20:36:36,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table
+2025-01-04 20:36:36,845 INFO    wandb-upload_1:2012081 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1d0g8ewq
+2025-01-04 20:36:36,881 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 20:36:36,890 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:36:36,895 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/qvy9l0ls-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:36,954 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/uk47cq8m-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:37,103 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:37,239 INFO    SenderThread:2012081 [sender.py:send_request_log_artifact():1525] logged artifact run-zsx5pj5g-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEwNjA1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg0OTY4Nw==', 'latestArtifact': None}}
+2025-01-04 20:36:37,239 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:37,239 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,239 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,240 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:36:37,240 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-04 20:36:37,240 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:37,240 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,243 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,243 DEBUG   SenderThread:2012081 [sender.py:send():379] send: telemetry
+2025-01-04 20:36:37,244 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,244 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,244 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,244 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,244 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,245 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,245 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,245 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,245 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,246 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,246 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,246 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,247 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,247 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,247 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,247 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,247 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,248 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,248 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,248 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,248 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,249 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,249 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 20:36:37,506 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/ghece49t-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:37,644 INFO    wandb-upload_0:2012081 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5ja0guqr
+2025-01-04 20:36:37,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval
+2025-01-04 20:36:38,005 INFO    SenderThread:2012081 [sender.py:send_request_log_artifact():1525] logged artifact run-zsx5pj5g-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEwNjA1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg0OTY4OA==', 'latestArtifact': None}}
+2025-01-04 20:36:38,005 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:36:38,005 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json with policy now
+2025-01-04 20:36:38,006 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:38,006 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:38,006 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:38,007 DEBUG   SenderThread:2012081 [sender.py:send():379] send: exit
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:send_exit():586] handling exit code: 0
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:send_exit():588] handling runtime: 278
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:send_exit():594] send defer
+2025-01-04 20:36:38,007 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,007 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-04 20:36:38,008 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 1
+2025-01-04 20:36:38,008 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-04 20:36:38,008 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 2
+2025-01-04 20:36:38,008 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-04 20:36:38,009 DEBUG   SystemMonitor:2012081 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-04 20:36:38,009 DEBUG   SystemMonitor:2012081 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-04 20:36:38,009 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined disk monitor
+2025-01-04 20:36:38,076 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-04 20:36:38,076 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined memory monitor
+2025-01-04 20:36:38,076 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined network monitor
+2025-01-04 20:36:38,076 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,076 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-04 20:36:38,077 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 3
+2025-01-04 20:36:38,077 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:36:38,077 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,077 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-04 20:36:38,077 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,077 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-04 20:36:38,077 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 4
+2025-01-04 20:36:38,077 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,077 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-04 20:36:38,184 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/b4vsq5qa-media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:38,425 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/3ucbmwwo-media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:38,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:39,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:44,406 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:44,406 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:44,407 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-04 20:36:44,407 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:44,408 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,408 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-04 20:36:44,408 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 5
+2025-01-04 20:36:44,408 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:44,408 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:44,408 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-04 20:36:44,408 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:44,409 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:44,409 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:44,409 DEBUG   SenderThread:2012081 [sender.py:send():379] send: summary
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:44,410 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 6
+2025-01-04 20:36:44,410 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:44,410 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-04 20:36:44,410 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-04 20:36:44,414 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:44,518 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 7
+2025-01-04 20:36:44,518 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:44,518 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-04 20:36:44,518 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,518 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-04 20:36:44,709 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
+2025-01-04 20:36:44,710 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:45,227 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:46,401 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 8
+2025-01-04 20:36:46,401 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:46,401 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:46,402 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-04 20:36:46,402 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:46,402 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-04 20:36:46,402 INFO    SenderThread:2012081 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-04 20:36:46,404 INFO    SenderThread:2012081 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-04 20:36:46,406 INFO    SenderThread:2012081 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-04 20:36:46,416 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 9
+2025-01-04 20:36:46,416 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:46,416 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-04 20:36:46,416 DEBUG   SenderThread:2012081 [sender.py:send():379] send: artifact
+2025-01-04 20:36:46,643 INFO    SenderThread:2012081 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'versionIndex': 416}}}
+2025-01-04 20:36:46,643 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:46,643 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-04 20:36:46,643 INFO    SenderThread:2012081 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml config.yaml
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json wandb-summary.json
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0 events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt requirements.txt
+2025-01-04 20:36:46,714 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json wandb-metadata.json
+2025-01-04 20:36:46,714 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log output.log
+2025-01-04 20:36:46,715 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch diff.patch
+2025-01-04 20:36:46,715 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4 media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4 media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4 media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4 media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4 media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 10
+2025-01-04 20:36:46,720 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:46,720 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-04 20:36:46,721 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:46,721 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-04 20:36:46,721 INFO    SenderThread:2012081 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 20:36:46,901 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
+2025-01-04 20:36:46,933 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:46,938 INFO    wandb-upload_3:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/70fq6v9l-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:46,965 INFO    wandb-upload_4:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:46,987 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt
+2025-01-04 20:36:47,187 INFO    Thread-11 (_thread_body):2012081 [sender.py:transition_state():614] send defer: 11
+2025-01-04 20:36:47,187 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,187 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-04 20:36:47,188 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,188 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-04 20:36:47,188 INFO    SenderThread:2012081 [file_pusher.py:join():175] waiting for file pusher
+2025-01-04 20:36:47,189 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 12
+2025-01-04 20:36:47,189 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,189 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-04 20:36:47,189 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,189 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-04 20:36:47,189 INFO    SenderThread:2012081 [file_stream.py:finish():601] file stream finish called
+2025-01-04 20:36:47,228 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [file_stream.py:finish():605] file stream finish is done
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 13
+2025-01-04 20:36:47,262 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:47,262 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,262 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-04 20:36:47,262 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 14
+2025-01-04 20:36:47,263 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,263 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-04 20:36:47,263 DEBUG   SenderThread:2012081 [sender.py:send():379] send: final
+2025-01-04 20:36:47,263 DEBUG   SenderThread:2012081 [sender.py:send():379] send: footer
+2025-01-04 20:36:47,263 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,263 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: server_info
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-04 20:36:47,265 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:47,265 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: server_info
+2025-01-04 20:36:47,266 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:47,315 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:47,315 INFO    MainThread:2012081 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-04 20:36:47,316 INFO    MainThread:2012081 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-04 20:36:47,316 INFO    MainThread:2012081 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-04 20:36:47,316 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-04 20:36:47,316 INFO    HandlerThread:2012081 [handler.py:finish():882] shutting down handler
+2025-01-04 20:36:48,265 INFO    WriterThread:2012081 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb
+2025-01-04 20:36:48,315 INFO    SenderThread:2012081 [sender.py:finish():1615] shutting down sender
+2025-01-04 20:36:48,315 INFO    SenderThread:2012081 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 20:36:48,315 INFO    SenderThread:2012081 [file_pusher.py:join():175] waiting for file pusher
diff --git a/zsx5pj5g/wandb/debug.log b/zsx5pj5g/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..423a962fbbbd770f7a5db961536ca063444aaa2f
--- /dev/null
+++ b/zsx5pj5g/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Configure stats pid to 2011907
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/settings
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug.log
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug-internal.log
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():569] calling init triggers
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g'}
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():619] starting backend
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():623] setting up manager
+2025-01-04 20:31:58,356 INFO    MainThread:2011907 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-04 20:31:58,357 INFO    MainThread:2011907 [wandb_init.py:init():631] backend started and connected
+2025-01-04 20:31:58,369 INFO    MainThread:2011907 [wandb_init.py:init():720] updated telemetry
+2025-01-04 20:31:58,375 INFO    MainThread:2011907 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-04 20:31:58,634 INFO    MainThread:2011907 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-04 20:31:58,673 INFO    MainThread:2011907 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-04 20:31:58,674 INFO    MainThread:2011907 [wandb_init.py:init():804] starting run threads in backend
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-04 20:31:59,021 INFO    MainThread:2011907 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-04 20:31:59,021 INFO    MainThread:2011907 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/tb, True
+2025-01-04 20:32:05,295 INFO    MainThread:2011907 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'}
+2025-01-04 20:36:48,317 WARNING MsgRouterThr:2011907 [router.py:message_loop():77] message_loop has been closed
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..350995e2b55a945bdaf88b9ca861134c7eb33382
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 1
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1930_0'
+wandb_notes:
+  desc: null
+  value: 1930_0_restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736040718
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: lib
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..f345052a4098779e6d1835354dda24725cd2d7e6
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch
@@ -0,0 +1,56 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 57a851f..7b7e871 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
++# wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1006_0'
++# wandb_run_id: '1926_1' # seed 1
++wandb_run_id: '1930_0' # dataset shuffle seed 1
++
++
+ 
+ # RGBD+mask+contact(contact+DTC)+act history
+-wandb_run_id: '1745_0'
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 71a5d1c..c7eadce 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -159,8 +159,10 @@ run_id_list = [
+ ]
+ 
+ run_id_list = [
+-    # '1006_0',
+-    '1745_0'
++    # '1922_1',
++    # '1922_2',
++    '1926_1',
++    '1930_0',
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py
+index 6f97e26..285afc8 100644
+--- a/gym-envs/gym_envs/envs/hardware/franka.py
++++ b/gym-envs/gym_envs/envs/hardware/franka.py
+@@ -330,8 +330,8 @@ class Franka:
+ 		# self.yaw = yaw
+ 
+ 		self.move_to_target_hz = 10.0
+-		self.move_to_trans_tolerance = 0.005
+-		self.move_to_rot_tolerance = 1.0
++		self.move_to_trans_tolerance = 0.01
++		self.move_to_rot_tolerance = 2.0
+ 		self.move_to_target_max_velocity = 0.8
+ 		self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz)
+ 		self.move_to_target_timeout = 5.0
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..7bcd112abb0564b1636cc9a424170abf69434e74
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[35, 40, 0.875, 0.7125501036218882, 0.9631809442309035, 0.7509347199891807, 0.95762218525794, 0.7729179039051675, 0.9400753284855695, 0.7853819727326881, 0.9362520258001197]]}
\ No newline at end of file
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4 b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..63abe916f8e9af4b906f0a76358f991eccffb437
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f959e9c50cf3c3a02940b1c093c5f31d1ee65c40cebd70655e74088c0110fc
+size 1499363
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4 b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7c98c27f9ae8ff7042954be0529e3033342950e0
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b2a0a5a222317bf8a15858c4651a4b33f187b761ddbe09d3cf9d26d948217a7
+size 1494961
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4 b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f12a700a0b428210999d362eaf5d47a20b99f9f9
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13d165b6bcf603a161c53a6adcc616c31483bd32f7b03d6d459b80355c0ea8c1
+size 1496842
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4 b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..00889217ac3f1a52f02e35dd8d949e300caa6125
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378a37ba58c54e86d1ce679bed3377ae8b329965e85371569be0e53c2a9a51f0
+size 1512946
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4 b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4d79217710a815541d32eb969593990a1ea5daf7
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a49e5d967d992227eafdc78f00824719e0350cf4dbfe490a10fe06331ff189
+size 1544024
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..6bf8ff22789673da038da8f812fa8ea1563f5a66
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
@@ -0,0 +1,277 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736040725.295613]: resetting environment
+[INFO] [1736040725.299732]: cleared current plan
+[INFO] [1736040725.300690]: moving to home
+[INFO] [1736040726.302815]: reached home
+[INFO] [1736040726.304256]: reset action history
+[INFO] [1736040729.016298]: environment reset
+Starting episode 0
+[INFO] [1736040729.021450]: resetting environment
+[INFO] [1736040729.025801]: cleared current plan
+[INFO] [1736040729.026642]: moving to home
+[INFO] [1736040730.028933]: reached home
+[INFO] [1736040730.030363]: reset action history
+[INFO] [1736040732.740514]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736040737.534926]: resetting environment
+[INFO] [1736040737.541470]: cleared current plan
+[INFO] [1736040737.542355]: moving to home
+[INFO] [1736040738.544895]: reached home
+[INFO] [1736040738.546471]: reset action history
+[INFO] [1736040741.305435]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/episode_rosbags/episode_0_2025-01-04-20-32-21.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2011907.log
+started roslaunch server http://158.130.50.37:37977/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2012188]
+process[print_text-2]: started with pid [2012189]
+process[rosbag_record-3]: started with pid [2012213]
+started bagging!
+[INFO] [1736040776.189451]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1736040776.970736]: Stored episode 1.
+[INFO] [1736040776.972576]: Saving video...
+[INFO] [1736040777.167177]: Video saved!
+Starting episode 1
+[INFO] [1736040777.186028]: resetting environment
+[INFO] [1736040777.194559]: cleared current plan
+[INFO] [1736040777.195086]: moving to home
+[INFO] [1736040780.897279]: reached home
+[INFO] [1736040780.898125]: reset action history
+[INFO] [1736040783.610279]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736040798.226745]: resetting environment
+[INFO] [1736040798.229859]: cleared current plan
+[INFO] [1736040798.230211]: moving to home
+[INFO] [1736040799.231704]: reached home
+[INFO] [1736040799.232133]: reset action history
+[INFO] [1736040801.938803]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/episode_rosbags/episode_1_2025-01-04-20-33-21.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2011907.log
+started roslaunch server http://158.130.50.37:46209/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2012337]
+process[print_text-5]: started with pid [2012361]
+process[rosbag_record-6]: started with pid [2012362]
+started bagging!
+For topic gripper_width: timestamp difference is -107408785 for nearest: 1736040806151579119 - target: 1736040806258987904 at idx 62
+For topic gripper_width: timestamp difference is 135587330 for nearest: 1736040818418268318 - target: 1736040818282680988 at idx 215
+For topic gripper_width: timestamp difference is 102674474 for nearest: 1736040818918245544 - target: 1736040818815571070 at idx 221
+[INFO] [1736040836.903602]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736040837.717191]: Stored episode 2.
+[INFO] [1736040837.718441]: Saving video...
+[INFO] [1736040838.399693]: Video saved!
+Starting episode 2
+[INFO] [1736040838.430225]: resetting environment
+[INFO] [1736040838.448593]: cleared current plan
+[INFO] [1736040838.449350]: moving to home
+[WARN] [1736040838.244909]: Plan exhausted
+[WARN] [1736040838.284374]: Plan exhausted
+[WARN] [1736040838.324665]: Plan exhausted
+[WARN] [1736040838.364416]: Plan exhausted
+[WARN] [1736040838.404472]: Plan exhausted
+[WARN] [1736040838.444924]: Plan exhausted
+[INFO] [1736040842.152374]: reached home
+[INFO] [1736040842.153767]: reset action history
+[INFO] [1736040844.878624]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736040846.591421]: resetting environment
+[INFO] [1736040846.604448]: cleared current plan
+[INFO] [1736040846.604685]: moving to home
+[INFO] [1736040847.608316]: reached home
+[INFO] [1736040847.608935]: reset action history
+[INFO] [1736040850.318503]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/episode_rosbags/episode_2_2025-01-04-20-34-10.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2011907.log
+started roslaunch server http://158.130.50.37:36641/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2012482]
+process[print_text-8]: started with pid [2012483]
+process[rosbag_record-9]: started with pid [2012484]
+started bagging!
+[INFO] [1736040885.228702]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit
+[print_text-8] killing on exit
+[INFO] [1736040886.007971]: Stored episode 3.
+[INFO] [1736040886.009444]: Saving video...
+[INFO] [1736040886.651172]: Video saved!
+Starting episode 3
+[INFO] [1736040886.692113]: resetting environment
+[INFO] [1736040886.716530]: cleared current plan
+[INFO] [1736040886.716779]: moving to home
+[WARN] [1736040886.565657]: Plan exhausted
+[WARN] [1736040886.605302]: Plan exhausted
+[WARN] [1736040886.645183]: Plan exhausted
+[WARN] [1736040886.684425]: Plan exhausted
+[INFO] [1736040890.425983]: reached home
+[INFO] [1736040890.430463]: reset action history
+[INFO] [1736040893.327851]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736040894.625168]: resetting environment
+[INFO] [1736040894.639971]: cleared current plan
+[INFO] [1736040894.642529]: moving to home
+[INFO] [1736040895.653862]: reached home
+[INFO] [1736040895.661112]: reset action history
+[INFO] [1736040898.404768]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/episode_rosbags/episode_3_2025-01-04-20-34-58.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2011907.log
+started roslaunch server http://158.130.50.37:41603/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2012646]
+process[print_text-11]: started with pid [2012647]
+process[rosbag_record-12]: started with pid [2012648]
+started bagging!
+For topic gripper_width: timestamp difference is 101846888 for nearest: 1736040900118263676 - target: 1736040900016416788 at idx 26
+[INFO] [1736040933.387384]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1736040934.226928]: Stored episode 4.
+[INFO] [1736040934.231318]: Saving video...
+[INFO] [1736040934.600237]: Video saved!
+Starting episode 4
+[INFO] [1736040934.650235]: resetting environment
+[INFO] [1736040934.677515]: cleared current plan
+[INFO] [1736040934.682243]: moving to home
+[INFO] [1736040938.387354]: reached home
+[INFO] [1736040938.393992]: reset action history
+[INFO] [1736040941.183991]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736040945.044636]: resetting environment
+[INFO] [1736040945.073181]: cleared current plan
+[INFO] [1736040945.077794]: moving to home
+[INFO] [1736040946.080748]: reached home
+[INFO] [1736040946.083145]: reset action history
+[INFO] [1736040948.858237]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/episode_rosbags/episode_4_2025-01-04-20-35-48.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2011907.log
+started roslaunch server http://158.130.50.37:43745/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2012788]
+process[print_text-14]: started with pid [2012789]
+process[rosbag_record-15]: started with pid [2012790]
+started bagging!
+[INFO] [1736040985.199726]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1736040986.039243]: Stored episode 5.
+[INFO] [1736040986.040798]: Saving video...
+[INFO] [1736040986.470661]: Video saved!
+[INFO] [1736040986.520468]: resetting environment
+[INFO] [1736040986.582907]: cleared current plan
+[INFO] [1736040986.583310]: moving to home
+[WARN] [1736040986.525216]: Plan exhausted
+[INFO] [1736040990.294041]: reached home
+[INFO] [1736040990.300003]: reset action history
+[INFO] [1736040993.101068]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..c61689c19339df5f2cb777a54a8b05095660d3c9
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-05T01:31:58.698894",
+    "startedAt": "2025-01-05T01:31:58.341612",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3749.790875,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.99,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 594.1672897338867
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..8979e5ba411f6cc335c42c93817ae2f8da290aa4
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "62f959e9c50cf3c3a02940b1c093c5f31d1ee65c40cebd70655e74088c0110fc", "size": 1499363, "path": "media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4"}, "global_step": 665, "_timestamp": 1736040996.8054287, "_runtime": 278.4480617046356, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "4b2a0a5a222317bf8a15858c4651a4b33f187b761ddbe09d3cf9d26d948217a7", "size": 1494961, "path": "media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4"}, "eval/num_success": 5.0, "episode": 40.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "13d165b6bcf603a161c53a6adcc616c31483bd32f7b03d6d459b80355c0ea8c1", "size": 1496842, "path": "media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "378a37ba58c54e86d1ce679bed3377ae8b329965e85371569be0e53c2a9a51f0", "size": 1512946, "path": "media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f8a49e5d967d992227eafdc78f00824719e0350cf4dbfe490a10fe06331ff189", "size": 1544024, "path": "media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://6x4rszyqnviggef9m6kky03hm3pdk3skihrq78y0ph707ta5ily2wksxy0ephtthanj91xxhr2fdp1xsjyrtbhroc5kanh2nk384yydjvih20uyp5c55b9zeelig970l/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qnshxifclswrlnpqp3s4bs677n4lcsvnlvakwf9lrl4808zeyp4eunyrxx0avw665n9zm9uz99qx7vakxhm93i6mnk3b7w56dki7yvot8zsywba01ttaipfoody7c04k:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 35, "total_num_episodes": 40, "total_success_rate": 0.875, "total_success_rate_lb_0.01": 0.7125501036218882, "total_success_rate_ub_0.01": 0.9631809442309035, "total_success_rate_lb_0.025": 0.7509347199891807, "total_success_rate_ub_0.025": 0.95762218525794, "total_success_rate_lb_0.05": 0.7729179039051675, "total_success_rate_ub_0.05": 0.9400753284855695, "total_success_rate_lb_0.1": 0.7853819727326881, "total_success_rate_ub_0.1": 0.9362520258001197, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "911e08243b29046ecf60c5f9e98382bed173b3da339f22dd1260a9b2c45849ed", "size": 505, "artifact_path": "wandb-client-artifact://78l9sbse4185km9o6mr426bx5t70myyb150as7davhxwbx2au2eh9wn8erf76dwdhb1kc8n37hbqc0yr1wao5as7n5qr856fng08oiklhjzw8j55qxesholbn5dhdm29/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://sgyr9wq519brc68jafzq575bxrt75s3iheczupefpz166muc8ut6zdeoqhqdy1qlez2jvtxk5inh0lg92mctygt8ajtm8itm7eghpzgnyfsxllb12m2l7tfa0viluzxa:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 278}, "eval/total_success_rate": 0.875, "eval/total_success_rate_lb_0.01": 0.7125501036643982, "eval/total_success_rate_ub_0.01": 0.9631809592247009, "eval/total_success_rate_lb_0.025": 0.7509347200393677, "eval/total_success_rate_ub_0.025": 0.9576221704483032, "eval/total_success_rate_lb_0.05": 0.7729179263114929, "eval/total_success_rate_ub_0.05": 0.9400753378868103, "eval/total_success_rate_lb_0.1": 0.7853819727897644, "eval/total_success_rate_ub_0.1": 0.9362519979476929}
\ No newline at end of file
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug-internal.log b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..8010c11fa23c3705c965fcc73fbe7c1063addddc
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug-internal.log
@@ -0,0 +1,779 @@
+2025-01-04 20:31:58,357 INFO    StreamThr :2012081 [internal.py:wandb_internal():85] W&B internal server running at pid: 2012081, started at: 2025-01-04 20:31:58.356937
+2025-01-04 20:31:58,358 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status
+2025-01-04 20:31:58,369 INFO    WriterThread:2012081 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb
+2025-01-04 20:31:58,369 DEBUG   SenderThread:2012081 [sender.py:send():379] send: header
+2025-01-04 20:31:58,378 DEBUG   SenderThread:2012081 [sender.py:send():379] send: run
+2025-01-04 20:31:58,612 INFO    SenderThread:2012081 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files
+2025-01-04 20:31:58,612 INFO    SenderThread:2012081 [sender.py:_start_run_threads():1188] run started: zsx5pj5g with start time 1736040718.357367
+2025-01-04 20:31:58,635 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: check_version
+2025-01-04 20:31:58,635 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: check_version
+2025-01-04 20:31:58,680 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: run_start
+2025-01-04 20:31:58,684 DEBUG   HandlerThread:2012081 [system_info.py:__init__():26] System info init
+2025-01-04 20:31:58,685 DEBUG   HandlerThread:2012081 [system_info.py:__init__():41] System info init done
+2025-01-04 20:31:58,685 INFO    HandlerThread:2012081 [system_monitor.py:start():194] Starting system monitor
+2025-01-04 20:31:58,685 INFO    SystemMonitor:2012081 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-04 20:31:58,685 INFO    HandlerThread:2012081 [system_monitor.py:probe():214] Collecting system info
+2025-01-04 20:31:58,686 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started cpu monitoring
+2025-01-04 20:31:58,687 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started disk monitoring
+2025-01-04 20:31:58,688 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started gpu monitoring
+2025-01-04 20:31:58,688 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started memory monitoring
+2025-01-04 20:31:58,688 INFO    SystemMonitor:2012081 [interfaces.py:start():188] Started network monitoring
+2025-01-04 20:31:58,698 DEBUG   HandlerThread:2012081 [system_info.py:probe():152] Probing system
+2025-01-04 20:31:58,700 DEBUG   HandlerThread:2012081 [system_info.py:_probe_git():137] Probing git
+2025-01-04 20:31:58,704 DEBUG   HandlerThread:2012081 [system_info.py:_probe_git():145] Probing git done
+2025-01-04 20:31:58,705 DEBUG   HandlerThread:2012081 [system_info.py:probe():200] Probing system done
+2025-01-04 20:31:58,705 DEBUG   HandlerThread:2012081 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:31:58.698894', 'startedAt': '2025-01-05T01:31:58.341612', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.790875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.99, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 594.1672897338867}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-04 20:31:58,705 INFO    HandlerThread:2012081 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-04 20:31:58,705 INFO    HandlerThread:2012081 [system_monitor.py:probe():227] Publishing system info
+2025-01-04 20:31:58,705 DEBUG   HandlerThread:2012081 [system_info.py:_save_code():44] Saving code
+2025-01-04 20:31:58,706 DEBUG   HandlerThread:2012081 [system_info.py:_save_code():67] Saving code done
+2025-01-04 20:31:58,707 DEBUG   HandlerThread:2012081 [system_info.py:_save_patches():84] Saving git patches
+2025-01-04 20:31:58,775 DEBUG   HandlerThread:2012081 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-04 20:31:58,775 INFO    HandlerThread:2012081 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-04 20:31:58,779 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:31:58,779 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-04 20:31:58,779 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-04 20:31:58,780 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-04 20:31:59,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-04 20:31:59,018 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: python_packages
+2025-01-04 20:31:59,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:31:59,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:31:59,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:31:59,021 INFO    HandlerThread:2012081 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/tb"
+  save: true
+}
+control {
+  relay_id: "5eecefe1ab4e42da905d511a8b6149f8"
+}
+_info {
+  stream_id: "zsx5pj5g"
+}
+
+2025-01-04 20:31:59,022 DEBUG   HandlerThread:2012081 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-04 20:31:59,060 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/9fe428ne-wandb-metadata.json
+2025-01-04 20:31:59,078 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/0w0bfaur-diff.patch
+2025-01-04 20:31:59,099 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/s42dfmkq-code/FISH/eval_robot.py
+2025-01-04 20:31:59,126 DEBUG   SenderThread:2012081 [sender.py:send():379] send: telemetry
+2025-01-04 20:31:59,126 DEBUG   SenderThread:2012081 [sender.py:send():379] send: tbrecord
+2025-01-04 20:31:59,127 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:31:59,127 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0 with policy live
+2025-01-04 20:31:59,309 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/91qfc23b-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json
+2025-01-04 20:31:59,614 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch
+2025-01-04 20:31:59,615 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH
+2025-01-04 20:31:59,615 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code
+2025-01-04 20:32:00,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:01,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:01,613 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:02,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:03,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:04,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:04,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:05,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:05,295 DEBUG   SenderThread:2012081 [sender.py:send():379] send: config
+2025-01-04 20:32:06,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:07,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:07,615 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:08,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:09,057 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:09,058 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:10,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:11,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:11,616 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:12,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:13,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:14,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:14,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:14,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:14,152 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:15,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:15,617 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:16,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:17,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:18,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:19,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:19,618 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:20,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:20,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:21,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:22,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:23,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:23,619 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:24,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:25,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:25,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:26,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:27,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:28,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:29,018 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:29,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:29,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:30,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:30,181 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:30,621 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
+2025-01-04 20:32:31,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:32,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:33,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:34,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:35,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:36,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:36,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:37,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:38,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:39,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:40,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:41,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:42,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:42,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:43,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:44,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:44,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:44,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:45,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:46,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:47,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:47,177 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:48,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:49,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:50,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:51,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:52,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:53,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:53,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:54,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:55,026 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:56,027 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:57,028 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:57,213 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:32:57,214 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:32:57,214 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4 with policy now
+2025-01-04 20:32:57,214 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:32:57,214 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:32:57,215 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:32:57,480 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/i9tx05it-media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media
+2025-01-04 20:32:57,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:32:57,894 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/okjmxkum-media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:32:58,028 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:58,215 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:32:58,689 DEBUG   SystemMonitor:2012081 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-04 20:32:58,689 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:32:59,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:32:59,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:32:59,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:32:59,629 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:00,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:01,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:02,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:03,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:03,610 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:03,631 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:04,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:05,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:05,632 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:06,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:07,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:08,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:09,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:09,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:10,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:11,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:12,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:13,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:14,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:14,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:14,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:14,162 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:15,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:16,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:17,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:18,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:18,636 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:33:18,871 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/lh46lxe0-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:33:19,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:19,232 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:19,636 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:20,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:21,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:21,637 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:22,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:23,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:23,637 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:24,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:25,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:25,032 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:26,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:27,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:27,639 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:28,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:28,690 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:33:29,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:29,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:29,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:30,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:30,134 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:31,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:32,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:33,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:34,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:35,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:36,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:36,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:37,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:38,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:39,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:39,643 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:40,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:41,022 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:41,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:41,643 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:33:42,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:43,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:44,019 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:44,019 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:44,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:45,034 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:46,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:46,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:47,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:48,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:49,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:50,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:51,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:52,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:52,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:53,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:54,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:55,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:56,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:57,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:57,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:33:58,036 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:58,422 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:33:58,422 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4 with policy now
+2025-01-04 20:33:58,425 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:33:58,425 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:33:58,426 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:33:58,428 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:33:58,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:33:58,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:33:58,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:33:58,688 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/dkat7zbw-media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:33:58,691 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:33:58,972 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/h513uszj-media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:33:59,023 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:33:59,024 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:33:59,038 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:33:59,650 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:00,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:00,651 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:01,037 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:02,046 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:02,102 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:03,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:03,652 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:04,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:05,038 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:06,038 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:06,654 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:34:07,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:07,448 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:34:07,448 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:34:07,448 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:07,448 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:34:07,654 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:07,654 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:34:08,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:09,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:09,655 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:10,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:11,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:11,655 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:12,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:12,565 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:13,039 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:14,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:14,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:14,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:15,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:16,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:17,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:18,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:18,092 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:19,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:20,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:21,040 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:22,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:23,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:24,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:24,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:25,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:26,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:27,041 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:28,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:28,691 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:34:29,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:29,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:29,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:29,076 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:30,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:31,042 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:32,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:33,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:34,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:35,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:35,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:36,043 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:37,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:38,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:39,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:40,025 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:40,044 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:41,045 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:42,045 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:43,045 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:44,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:44,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:44,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:45,046 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:45,106 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:46,049 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:46,684 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:34:46,685 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4 with policy now
+2025-01-04 20:34:46,687 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:34:46,688 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:34:46,688 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:34:46,695 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:34:46,973 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/m7is78bd-media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:34:47,048 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:47,667 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:47,667 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:34:47,667 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:34:47,668 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:34:47,982 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/s93af7zb-media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:34:48,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:48,668 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:49,050 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:50,055 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:50,427 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:51,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:51,669 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:52,057 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:53,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:54,054 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:54,670 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:34:54,902 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/yyioaer2-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:34:55,070 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:55,284 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:34:55,285 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:34:55,285 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:34:55,659 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:34:55,670 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:55,670 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:34:56,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:57,070 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:57,671 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:34:58,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:58,692 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:34:59,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:34:59,020 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:34:59,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:34:59,672 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:00,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:01,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:01,173 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:01,672 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:02,059 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:03,060 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:04,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:05,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:06,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:07,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:07,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:08,061 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:09,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:10,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:11,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:12,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:13,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:13,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:14,020 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:14,021 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:14,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:15,062 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:16,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:17,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:18,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:18,129 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:19,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:20,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:21,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:22,063 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:23,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:24,031 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:24,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:25,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:26,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:27,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:28,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:28,692 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:35:29,021 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:29,021 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:29,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:29,103 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:30,064 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:31,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:32,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:33,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:34,071 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:34,228 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:34,645 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:35:34,645 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4 with policy now
+2025-01-04 20:35:34,648 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:35:34,649 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:35:34,649 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:35:34,669 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:35:34,683 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:35:34,683 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:35:34,684 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:35:34,936 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/t1f5m9tn-media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:35:35,065 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:35,204 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/hojre09c-media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:35:35,684 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:36,066 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:37,073 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:38,071 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:39,078 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:39,685 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:40,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:40,072 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:41,073 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:42,075 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:43,087 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:43,686 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:44,029 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:44,029 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:44,086 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:45,066 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:45,076 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:45,342 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:35:45,342 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:35:45,343 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:35:45,687 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:35:45,687 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:35:46,077 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:47,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:47,688 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:48,088 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:49,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:49,688 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:50,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:50,271 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:51,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:51,689 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:35:52,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:53,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:54,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:55,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:56,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:35:56,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:57,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:58,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:35:58,693 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:35:59,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:35:59,024 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:35:59,079 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:00,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:01,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:01,148 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:02,081 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:03,080 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:04,081 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:05,082 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:06,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:07,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:07,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:08,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:09,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:10,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:11,083 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:12,033 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:12,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:13,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:14,024 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:36:14,024 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:36:14,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:15,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:16,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:17,084 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:17,129 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:18,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:19,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:20,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:21,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:22,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:23,035 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:23,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:24,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:25,085 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:26,091 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:26,513 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:36:26,513 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4 with policy now
+2025-01-04 20:36:26,514 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:36:26,514 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:26,514 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:26,521 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:26,702 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:26,702 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:26,703 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval
+2025-01-04 20:36:26,788 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/6xzu7tyn-media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:27,036 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/aybqvku0-media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:27,095 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:27,703 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:28,099 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:28,583 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:28,693 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:36:28,703 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:29,030 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-04 20:36:29,030 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: stop_status
+2025-01-04 20:36:29,098 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:30,098 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:31,101 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:31,704 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:32,101 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:33,145 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:34,048 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:34,103 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:35,110 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:35,705 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:36,108 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:36,370 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 20:36:36,370 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 20:36:36,499 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:36:36,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:36,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:36,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media
+2025-01-04 20:36:36,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval
+2025-01-04 20:36:36,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table
+2025-01-04 20:36:36,845 INFO    wandb-upload_1:2012081 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1d0g8ewq
+2025-01-04 20:36:36,881 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-04 20:36:36,890 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-04 20:36:36,895 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/qvy9l0ls-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:36,954 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/uk47cq8m-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:37,103 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:37,239 INFO    SenderThread:2012081 [sender.py:send_request_log_artifact():1525] logged artifact run-zsx5pj5g-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEwNjA1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg0OTY4Nw==', 'latestArtifact': None}}
+2025-01-04 20:36:37,239 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:37,239 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,239 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,240 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:36:37,240 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-04 20:36:37,240 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:37,240 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,243 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,243 DEBUG   SenderThread:2012081 [sender.py:send():379] send: telemetry
+2025-01-04 20:36:37,244 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,244 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,244 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,244 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,244 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,245 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,245 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,245 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,245 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,246 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,246 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,246 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,247 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,247 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,247 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,247 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,247 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,248 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,248 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,248 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,248 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:37,249 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:37,249 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: log_artifact
+2025-01-04 20:36:37,506 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/ghece49t-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:37,644 INFO    wandb-upload_0:2012081 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5ja0guqr
+2025-01-04 20:36:37,706 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:37,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval
+2025-01-04 20:36:38,005 INFO    SenderThread:2012081 [sender.py:send_request_log_artifact():1525] logged artifact run-zsx5pj5g-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEwNjA1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg0OTY4OA==', 'latestArtifact': None}}
+2025-01-04 20:36:38,005 DEBUG   SenderThread:2012081 [sender.py:send():379] send: files
+2025-01-04 20:36:38,005 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json with policy now
+2025-01-04 20:36:38,006 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:38,006 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:38,006 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:38,007 DEBUG   SenderThread:2012081 [sender.py:send():379] send: exit
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:send_exit():586] handling exit code: 0
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:send_exit():588] handling runtime: 278
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:38,007 INFO    SenderThread:2012081 [sender.py:send_exit():594] send defer
+2025-01-04 20:36:38,007 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,007 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-04 20:36:38,008 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 1
+2025-01-04 20:36:38,008 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-04 20:36:38,008 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-04 20:36:38,008 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 2
+2025-01-04 20:36:38,008 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-04 20:36:38,008 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-04 20:36:38,009 DEBUG   SystemMonitor:2012081 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-04 20:36:38,009 DEBUG   SystemMonitor:2012081 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-04 20:36:38,009 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined disk monitor
+2025-01-04 20:36:38,076 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-04 20:36:38,076 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined memory monitor
+2025-01-04 20:36:38,076 INFO    HandlerThread:2012081 [interfaces.py:finish():200] Joined network monitor
+2025-01-04 20:36:38,076 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,076 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-04 20:36:38,077 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 3
+2025-01-04 20:36:38,077 DEBUG   SenderThread:2012081 [sender.py:send():379] send: stats
+2025-01-04 20:36:38,077 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,077 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-04 20:36:38,077 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:38,077 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-04 20:36:38,077 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 4
+2025-01-04 20:36:38,077 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:38,077 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-04 20:36:38,184 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/b4vsq5qa-media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:38,425 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/3ucbmwwo-media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:38,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:39,707 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:44,406 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:44,406 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:44,407 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-04 20:36:44,407 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:44,408 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,408 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-04 20:36:44,408 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 5
+2025-01-04 20:36:44,408 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:44,408 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:44,408 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-04 20:36:44,408 DEBUG   SenderThread:2012081 [sender.py:send():379] send: history
+2025-01-04 20:36:44,409 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: summary_record
+2025-01-04 20:36:44,409 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:44,409 DEBUG   SenderThread:2012081 [sender.py:send():379] send: summary
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-04 20:36:44,410 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 6
+2025-01-04 20:36:44,410 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:44,410 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-04 20:36:44,410 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,410 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-04 20:36:44,414 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: status_report
+2025-01-04 20:36:44,518 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 7
+2025-01-04 20:36:44,518 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:44,518 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-04 20:36:44,518 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:44,518 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-04 20:36:44,709 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
+2025-01-04 20:36:44,710 INFO    Thread-12 :2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:45,227 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:46,401 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 8
+2025-01-04 20:36:46,401 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:46,401 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:46,402 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-04 20:36:46,402 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:46,402 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-04 20:36:46,402 INFO    SenderThread:2012081 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-04 20:36:46,404 INFO    SenderThread:2012081 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-04 20:36:46,406 INFO    SenderThread:2012081 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-04 20:36:46,416 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 9
+2025-01-04 20:36:46,416 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:46,416 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-04 20:36:46,416 DEBUG   SenderThread:2012081 [sender.py:send():379] send: artifact
+2025-01-04 20:36:46,643 INFO    SenderThread:2012081 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'versionIndex': 416}}}
+2025-01-04 20:36:46,643 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:46,643 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-04 20:36:46,643 INFO    SenderThread:2012081 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml config.yaml
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json wandb-summary.json
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0 events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:46,711 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt requirements.txt
+2025-01-04 20:36:46,714 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-metadata.json wandb-metadata.json
+2025-01-04 20:36:46,714 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log output.log
+2025-01-04 20:36:46,715 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/diff.patch diff.patch
+2025-01-04 20:36:46,715 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4 media/videos/eval/1_eval_1_4b2a0a5a222317bf8a15.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4 media/videos/eval/0_eval_0_62f959e9c50cf3c3a029.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4 media/videos/eval/4_eval_4_f8a49e5d967d992227ea.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4 media/videos/eval/2_eval_2_13d165b6bcf603a161c5.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4 media/videos/eval/3_eval_3_378a37ba58c54e86d1ce.mp4
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json media/table/eval/total_success_rate_ci_6_911e08243b29046ecf60.table.json
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-04 20:36:46,720 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 10
+2025-01-04 20:36:46,720 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:46,720 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-04 20:36:46,721 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:46,721 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-04 20:36:46,721 INFO    SenderThread:2012081 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 20:36:46,901 INFO    wandb-upload_2:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/config.yaml
+2025-01-04 20:36:46,933 INFO    wandb-upload_1:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/wandb-summary.json
+2025-01-04 20:36:46,938 INFO    wandb-upload_3:2012081 [upload_job.py:push():130] Uploaded file /tmp/tmptg1sr64ywandb/70fq6v9l-events.out.tfevents.1736040719.leonmkim-ROG-Strix-G15CS-G15CS.2011907.0
+2025-01-04 20:36:46,965 INFO    wandb-upload_4:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/output.log
+2025-01-04 20:36:46,987 INFO    wandb-upload_0:2012081 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/files/requirements.txt
+2025-01-04 20:36:47,187 INFO    Thread-11 (_thread_body):2012081 [sender.py:transition_state():614] send defer: 11
+2025-01-04 20:36:47,187 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,187 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-04 20:36:47,188 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,188 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-04 20:36:47,188 INFO    SenderThread:2012081 [file_pusher.py:join():175] waiting for file pusher
+2025-01-04 20:36:47,189 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 12
+2025-01-04 20:36:47,189 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,189 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-04 20:36:47,189 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,189 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-04 20:36:47,189 INFO    SenderThread:2012081 [file_stream.py:finish():601] file stream finish called
+2025-01-04 20:36:47,228 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [file_stream.py:finish():605] file stream finish is done
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 13
+2025-01-04 20:36:47,262 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:47,262 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,262 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-04 20:36:47,262 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-04 20:36:47,262 INFO    SenderThread:2012081 [sender.py:transition_state():614] send defer: 14
+2025-01-04 20:36:47,263 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: defer
+2025-01-04 20:36:47,263 INFO    HandlerThread:2012081 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-04 20:36:47,263 DEBUG   SenderThread:2012081 [sender.py:send():379] send: final
+2025-01-04 20:36:47,263 DEBUG   SenderThread:2012081 [sender.py:send():379] send: footer
+2025-01-04 20:36:47,263 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: defer
+2025-01-04 20:36:47,263 INFO    SenderThread:2012081 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: server_info
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-04 20:36:47,264 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-04 20:36:47,265 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:47,265 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: server_info
+2025-01-04 20:36:47,266 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-04 20:36:47,315 DEBUG   SenderThread:2012081 [sender.py:send_request():406] send_request: poll_exit
+2025-01-04 20:36:47,315 INFO    MainThread:2012081 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-04 20:36:47,316 INFO    MainThread:2012081 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-04 20:36:47,316 INFO    MainThread:2012081 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-04 20:36:47,316 DEBUG   HandlerThread:2012081 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-04 20:36:47,316 INFO    HandlerThread:2012081 [handler.py:finish():882] shutting down handler
+2025-01-04 20:36:48,265 INFO    WriterThread:2012081 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb
+2025-01-04 20:36:48,315 INFO    SenderThread:2012081 [sender.py:finish():1615] shutting down sender
+2025-01-04 20:36:48,315 INFO    SenderThread:2012081 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-04 20:36:48,315 INFO    SenderThread:2012081 [file_pusher.py:join():175] waiting for file pusher
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug.log b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..423a962fbbbd770f7a5db961536ca063444aaa2f
--- /dev/null
+++ b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Configure stats pid to 2011907
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/settings
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-04 20:31:58,349 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug.log
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/logs/debug-internal.log
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():569] calling init triggers
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g'}
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():619] starting backend
+2025-01-04 20:31:58,350 INFO    MainThread:2011907 [wandb_init.py:init():623] setting up manager
+2025-01-04 20:31:58,356 INFO    MainThread:2011907 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-04 20:31:58,357 INFO    MainThread:2011907 [wandb_init.py:init():631] backend started and connected
+2025-01-04 20:31:58,369 INFO    MainThread:2011907 [wandb_init.py:init():720] updated telemetry
+2025-01-04 20:31:58,375 INFO    MainThread:2011907 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-04 20:31:58,634 INFO    MainThread:2011907 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-04 20:31:58,673 INFO    MainThread:2011907 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-04 20:31:58,674 INFO    MainThread:2011907 [wandb_init.py:init():804] starting run threads in backend
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-04 20:31:59,019 INFO    MainThread:2011907 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-04 20:31:59,021 INFO    MainThread:2011907 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-04 20:31:59,021 INFO    MainThread:2011907 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/zsx5pj5g/tb, True
+2025-01-04 20:32:05,295 INFO    MainThread:2011907 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'}
+2025-01-04 20:36:48,317 WARNING MsgRouterThr:2011907 [router.py:message_loop():77] message_loop has been closed
diff --git a/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..c4e3bc7c53d20371b6e4647220da91561f907b1e
Binary files /dev/null and b/zsx5pj5g/wandb/run-20250104_203158-zsx5pj5g/run-zsx5pj5g.wandb differ