diff --git a/.gitattributes b/.gitattributes
index 235d68d9d665c8a30fb14ed62f2b187b9fb16e09..a9d90db7e528798aaec019863a716d25a679420b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -107,3 +107,32 @@ no798ka2/episode_rosbags/episode_0_2025-01-07-18-23-32.bag filter=lfs diff=lfs m
 vhqg7j1c/episode_rosbags/episode_4_2025-01-06-17-59-06.bag filter=lfs diff=lfs merge=lfs -text
 w4b02c7s/episode_rosbags/episode_4_2025-01-07-20-24-46.bag filter=lfs diff=lfs merge=lfs -text
 w4b02c7s/episode_rosbags/episode_2_2025-01-07-20-22-57.bag filter=lfs diff=lfs merge=lfs -text
+vhqg7j1c/episode_rosbags/episode_2_2025-01-06-17-57-22.bag filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+no798ka2/episode_rosbags/episode_4_2025-01-07-18-27-12.bag filter=lfs diff=lfs merge=lfs -text
+73h7fwcu/episode_rosbags/episode_4_2025-01-07-16-44-16.bag filter=lfs diff=lfs merge=lfs -text
+w4b02c7s/episode_rosbags/episode_3_2025-01-07-20-23-51.bag filter=lfs diff=lfs merge=lfs -text
+vhqg7j1c/episode_rosbags/episode_0_2025-01-06-17-55-45.bag filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+zk6ipw9e/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bbomq0jb/episode_rosbags/episode_1_2025-01-06-22-47-52.bag filter=lfs diff=lfs merge=lfs -text
+0a7h4b5r/episode_rosbags/episode_3_2025-01-06-20-13-48.bag filter=lfs diff=lfs merge=lfs -text
+0a7h4b5r/episode_rosbags/episode_2_2025-01-06-20-12-57.bag filter=lfs diff=lfs merge=lfs -text
+0a7h4b5r/episode_rosbags/episode_1_2025-01-06-20-12-05.bag filter=lfs diff=lfs merge=lfs -text
diff --git a/0a7h4b5r/episode_rosbags/episode_1_2025-01-06-20-12-05.bag b/0a7h4b5r/episode_rosbags/episode_1_2025-01-06-20-12-05.bag
new file mode 100644
index 0000000000000000000000000000000000000000..7ef95b338f50e4a4ac3906dcf4693427e8e36846
--- /dev/null
+++ b/0a7h4b5r/episode_rosbags/episode_1_2025-01-06-20-12-05.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a14651c6c464c71364360f0f8b4ab52c1f0daa4a5e767e677c5180949a2abd59
+size 2597806871
diff --git a/0a7h4b5r/episode_rosbags/episode_2_2025-01-06-20-12-57.bag b/0a7h4b5r/episode_rosbags/episode_2_2025-01-06-20-12-57.bag
new file mode 100644
index 0000000000000000000000000000000000000000..0d4ba9a201bdd35761ed12f43c875a60b61a2c4d
--- /dev/null
+++ b/0a7h4b5r/episode_rosbags/episode_2_2025-01-06-20-12-57.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66edd3737cb8018ae250ef896ef6d76ba5ea8cb95b2ac029eb29a0e6cf969da2
+size 2597291659
diff --git a/0a7h4b5r/episode_rosbags/episode_3_2025-01-06-20-13-48.bag b/0a7h4b5r/episode_rosbags/episode_3_2025-01-06-20-13-48.bag
new file mode 100644
index 0000000000000000000000000000000000000000..38f2fad4ec2e1f850ccdb56b826c5aa8af8b1a43
--- /dev/null
+++ b/0a7h4b5r/episode_rosbags/episode_3_2025-01-06-20-13-48.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c3e464f9cf69088288b1e0b4f6a64e7136626eaa29dcdc49f2e16046cfafd73
+size 2600029213
diff --git a/7332ua2l/episode_rosbags/cam_tf_world.npy b/7332ua2l/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/7332ua2l/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/7332ua2l/episode_rosbags/depth_K.npy b/7332ua2l/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/7332ua2l/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/7332ua2l/eval_robot.log b/7332ua2l/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..869586df4b65de6cf099a098c2363eb3f2809f93
--- /dev/null
+++ b/7332ua2l/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-06 18:00:15,486][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 18:00:15,489][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/73h7fwcu/episode_rosbags/episode_4_2025-01-07-16-44-16.bag b/73h7fwcu/episode_rosbags/episode_4_2025-01-07-16-44-16.bag
new file mode 100644
index 0000000000000000000000000000000000000000..f21afacc44e6b2e50dc474d7ebfbda8980737ece
--- /dev/null
+++ b/73h7fwcu/episode_rosbags/episode_4_2025-01-07-16-44-16.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4812c2bddd20adf9b9c085911d98fd21627fbade9c3e941b8bc5c842b04e16b1
+size 2608190995
diff --git a/bbomq0jb/.hydra/config.yaml b/bbomq0jb/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3
--- /dev/null
+++ b/bbomq0jb/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1013_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/bbomq0jb/.hydra/hydra.yaml b/bbomq0jb/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fa23fd05aa12dd7232e0653b4e94efefcbc5cd59
--- /dev/null
+++ b/bbomq0jb/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/bbomq0jb/.hydra/overrides.yaml b/bbomq0jb/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/bbomq0jb/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/bbomq0jb/episode_rosbags/episode_1_2025-01-06-22-47-52.bag b/bbomq0jb/episode_rosbags/episode_1_2025-01-06-22-47-52.bag
new file mode 100644
index 0000000000000000000000000000000000000000..6c34ad7dbd6c52a5ff277cb5c5f42e4ae6712340
--- /dev/null
+++ b/bbomq0jb/episode_rosbags/episode_1_2025-01-06-22-47-52.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db142d294fcb65a132e1de98dbf5cbc22e6858d53143ae7a8ebb9d5111896a98
+size 2604791243
diff --git a/bbomq0jb/eval_video/0_eval.mp4 b/bbomq0jb/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f900f90d764c65723a4be539e3164a43fc3b380b
--- /dev/null
+++ b/bbomq0jb/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a689b02b00d8ffc5d19ed66c1a3d6e28e967f2c362567356590ff20300b76d8
+size 1491278
diff --git a/bbomq0jb/eval_video/1_eval.mp4 b/bbomq0jb/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..37229191a873cf7bd7601e48ed7414cd64656f81
--- /dev/null
+++ b/bbomq0jb/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75f06614e77b624267b20ffb66ebd2655692b88759d0376af598c505ff89a322
+size 1484012
diff --git a/bbomq0jb/eval_video/2_eval.mp4 b/bbomq0jb/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..b7e9cf7c016a575498c26beea7bb1ddad1f15612
--- /dev/null
+++ b/bbomq0jb/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e12dd92f9893feab8257bff3e23a9443c6ce289e4bf6aead5246edcb4012d1c
+size 1477285
diff --git a/bbomq0jb/eval_video/3_eval.mp4 b/bbomq0jb/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..43011bbd0c2404c83f1069614a206ac940ccee16
--- /dev/null
+++ b/bbomq0jb/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9e68fb8bbd70aad7917593a8d03a4b2eec5cc175bbf82ba089da2082f4b21b6
+size 1544991
diff --git a/bbomq0jb/eval_video/4_eval.mp4 b/bbomq0jb/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4486e22b47a0f0502a42855db66d58ebdb9ba061
--- /dev/null
+++ b/bbomq0jb/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e71831cf69946749641f10b8512e62d5485ff6d10187e87cc522e0f524e9573
+size 1516284
diff --git a/bbomq0jb/tb/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0 b/bbomq0jb/tb/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
new file mode 100644
index 0000000000000000000000000000000000000000..cd1b9866eaeae89427c8d6d88eea937f11fb5e52
--- /dev/null
+++ b/bbomq0jb/tb/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdbe9eb95f777c2151005be29f0e9faea4444d334baa10e224fa50ce5074588d
+size 6539
diff --git a/bbomq0jb/wandb/debug-internal.log b/bbomq0jb/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..5d2f3003c5eca0240cd1da7a8a511407d1008c4d
--- /dev/null
+++ b/bbomq0jb/wandb/debug-internal.log
@@ -0,0 +1,814 @@
+2025-01-06 22:46:34,435 INFO    StreamThr :2129923 [internal.py:wandb_internal():85] W&B internal server running at pid: 2129923, started at: 2025-01-06 22:46:34.434866
+2025-01-06 22:46:34,436 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status
+2025-01-06 22:46:34,447 INFO    WriterThread:2129923 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb
+2025-01-06 22:46:34,448 DEBUG   SenderThread:2129923 [sender.py:send():379] send: header
+2025-01-06 22:46:34,458 DEBUG   SenderThread:2129923 [sender.py:send():379] send: run
+2025-01-06 22:46:34,695 INFO    SenderThread:2129923 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files
+2025-01-06 22:46:34,695 INFO    SenderThread:2129923 [sender.py:_start_run_threads():1188] run started: bbomq0jb with start time 1736221594.435095
+2025-01-06 22:46:34,716 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 22:46:34,716 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: check_version
+2025-01-06 22:46:34,763 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 22:46:34,768 DEBUG   HandlerThread:2129923 [system_info.py:__init__():26] System info init
+2025-01-06 22:46:34,768 DEBUG   HandlerThread:2129923 [system_info.py:__init__():41] System info init done
+2025-01-06 22:46:34,768 INFO    HandlerThread:2129923 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 22:46:34,768 INFO    SystemMonitor:2129923 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 22:46:34,769 INFO    HandlerThread:2129923 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 22:46:34,769 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 22:46:34,770 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 22:46:34,770 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 22:46:34,771 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 22:46:34,771 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started network monitoring
+2025-01-06 22:46:34,781 DEBUG   HandlerThread:2129923 [system_info.py:probe():152] Probing system
+2025-01-06 22:46:34,783 DEBUG   HandlerThread:2129923 [system_info.py:_probe_git():137] Probing git
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_info.py:probe():200] Probing system done
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:46:34.781844', 'startedAt': '2025-01-07T03:46:34.427712', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.05225, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4498.855, 'min': 800.0, 'max': 4700.0}, {'current': 4498.576, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.287, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 614.3022308349609}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 22:46:34,788 INFO    HandlerThread:2129923 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 22:46:34,788 INFO    HandlerThread:2129923 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_info.py:_save_code():44] Saving code
+2025-01-06 22:46:34,789 DEBUG   HandlerThread:2129923 [system_info.py:_save_code():67] Saving code done
+2025-01-06 22:46:34,789 DEBUG   HandlerThread:2129923 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 22:46:34,854 DEBUG   HandlerThread:2129923 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 22:46:34,854 INFO    HandlerThread:2129923 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 22:46:34,857 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:46:34,857 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 22:46:34,858 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 22:46:34,858 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 22:46:35,093 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/glhoyajj-wandb-metadata.json
+2025-01-06 22:46:35,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 22:46:35,104 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 22:46:35,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:46:35,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:46:35,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:35,106 INFO    HandlerThread:2129923 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/tb"
+  save: true
+}
+control {
+  relay_id: "cfd43c4364754628a34595dcf6569b62"
+}
+_info {
+  stream_id: "bbomq0jb"
+}
+
+2025-01-06 22:46:35,107 DEBUG   HandlerThread:2129923 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 22:46:35,127 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/mco025zc-diff.patch
+2025-01-06 22:46:35,132 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/6qzbs4ta-code/FISH/eval_robot.py
+2025-01-06 22:46:35,164 DEBUG   SenderThread:2129923 [sender.py:send():379] send: telemetry
+2025-01-06 22:46:35,164 DEBUG   SenderThread:2129923 [sender.py:send():379] send: tbrecord
+2025-01-06 22:46:35,164 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:46:35,164 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0 with policy live
+2025-01-06 22:46:35,335 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/me5c1w3s-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:46:35,697 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json
+2025-01-06 22:46:35,697 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code
+2025-01-06 22:46:36,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:37,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:37,696 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:38,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:39,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:39,660 DEBUG   SenderThread:2129923 [sender.py:send():379] send: config
+2025-01-06 22:46:39,660 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:40,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:41,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:41,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:42,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:43,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:43,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:44,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:45,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:45,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:45,699 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:46,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:47,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:48,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:49,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:49,700 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:50,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:46:50,104 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:46:50,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:50,196 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:51,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:52,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:53,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:53,701 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:54,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:55,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:55,702 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:55,827 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:56,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:57,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:57,703 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:58,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:59,117 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:00,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:01,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:01,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:02,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:03,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:04,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:05,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:05,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:05,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:06,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:06,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:06,705 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
+2025-01-06 22:47:07,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:08,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:09,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:10,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:11,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:11,960 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:12,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:13,117 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:13,707 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:14,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:15,117 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:15,707 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:16,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:17,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:17,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:18,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:19,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:20,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:20,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:20,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:21,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:22,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:22,304 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:23,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:24,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:25,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:26,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:27,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:28,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:28,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:29,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:30,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:31,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:31,516 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:47:31,516 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4 with policy now
+2025-01-06 22:47:31,516 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:47:31,517 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:47:31,518 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:47:31,518 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media
+2025-01-06 22:47:31,777 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/0wxi5wtb-media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:47:32,044 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/eeh36p89-media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:47:32,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:33,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:33,538 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:33,713 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:34,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:34,771 DEBUG   SystemMonitor:2129923 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 22:47:34,772 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:47:35,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:35,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:35,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:36,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:37,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:37,714 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:38,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:38,901 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:39,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:39,715 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:40,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:41,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:42,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:43,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:44,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:44,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:45,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:46,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:47,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:48,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:49,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:49,718 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:47:49,853 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:49,996 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/huzfsfha-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:47:50,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:50,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:50,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:51,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:51,718 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:52,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:53,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:53,719 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:54,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:55,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:55,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:55,720 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:56,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:57,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:58,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:59,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:00,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:00,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:01,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:02,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:03,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:04,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:04,772 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:48:05,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:05,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:05,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:05,184 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:06,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:07,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:07,724 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:08,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:09,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:10,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:11,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:11,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:12,127 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:13,127 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:14,127 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:15,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:16,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:16,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:17,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:18,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:19,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:20,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:20,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:20,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:21,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:21,221 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:22,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:23,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:24,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:25,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:26,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:27,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:27,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:28,130 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:28,676 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:48:28,676 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4 with policy now
+2025-01-06 22:48:28,679 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:48:28,679 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:48:28,679 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:48:28,684 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:48:28,731 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:48:28,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:48:28,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:48:28,936 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/mgba4a4g-media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:48:29,130 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:29,208 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/o37dwl2w-media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:48:29,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:30,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:30,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:31,138 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:32,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:32,140 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:33,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:33,733 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:34,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:34,773 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:48:35,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:35,108 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:35,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:36,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:37,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:37,245 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:37,734 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:38,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:39,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:40,140 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:41,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:42,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:43,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:43,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:44,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:45,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:46,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:47,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:47,540 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:48:47,540 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:48:47,541 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:48:47,738 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:48:47,738 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:48:48,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:49,037 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:49,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:49,738 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:50,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:50,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:50,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:51,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:51,739 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:52,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:53,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:53,740 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:54,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:54,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:55,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:56,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:57,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:58,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:59,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:00,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:00,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:01,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:02,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:03,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:04,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:04,773 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:49:05,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:05,106 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:05,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:05,216 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:06,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:07,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:08,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:09,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:10,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:11,111 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:11,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:12,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:13,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:14,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:15,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:16,111 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:16,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:17,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:18,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:19,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:20,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:20,106 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:20,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:21,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:21,195 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:22,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:23,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:24,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:25,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:25,751 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:26,150 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:26,445 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:26,922 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:49:26,922 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4 with policy now
+2025-01-06 22:49:26,924 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:49:26,925 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:49:26,925 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:49:26,925 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:49:27,151 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:27,172 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/u2gyat7g-media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:49:27,751 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:49:27,751 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:27,752 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:49:27,752 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:49:28,045 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/xc5pw4o9-media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:49:28,154 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:29,156 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:29,752 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:30,156 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:31,154 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:31,593 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:31,753 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:32,153 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:33,155 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:34,154 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:34,774 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:49:35,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:35,110 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:35,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:35,754 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:36,158 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:36,371 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:49:36,371 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:49:36,371 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:49:36,754 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:49:36,755 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:49:36,998 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/12hzf77q-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:49:37,158 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:37,409 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:37,755 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:38,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:39,158 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:39,755 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:40,197 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:41,163 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:41,757 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:42,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:42,696 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:43,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:43,758 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:44,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:45,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:46,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:47,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:47,759 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:47,833 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:48,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:49,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:50,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:50,106 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:50,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:51,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:52,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:53,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:53,226 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:54,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:55,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:56,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:57,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:58,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:59,009 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:59,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:59,763 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:00,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:01,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:02,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:03,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:04,113 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:04,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:04,774 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:50:05,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:05,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:05,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:06,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:07,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:08,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:09,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:09,231 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:10,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:11,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:12,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:13,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:14,163 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:15,113 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:15,163 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:16,193 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:17,167 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:17,317 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:50:17,317 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4 with policy now
+2025-01-06 22:50:17,318 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:50:17,319 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:50:17,319 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:50:17,341 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:50:17,582 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/unzssuoz-media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:50:17,769 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:50:17,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:17,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:50:17,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:50:18,038 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/sn86bu0l-media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:50:18,172 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:19,182 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:19,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:20,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:20,109 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:20,175 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:20,184 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:21,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:21,771 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:22,179 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:23,183 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:24,181 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:25,193 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:25,772 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:26,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:26,181 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:27,183 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:27,438 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:50:27,438 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:50:27,438 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:50:27,773 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:27,773 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:50:27,773 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:50:28,183 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:29,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:29,774 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:30,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:31,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:31,774 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:31,983 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:32,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:33,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:33,775 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:34,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:34,775 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:50:35,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:35,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:35,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:36,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:37,184 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:37,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:38,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:39,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:40,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:41,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:42,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:43,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:43,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:44,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:45,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:45,779 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:46,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:47,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:47,780 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:48,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:48,313 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:49,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:50,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:50,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:50,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:51,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:52,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:53,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:54,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:54,249 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:55,188 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:56,213 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:57,188 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:57,783 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:58,189 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:59,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:00,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:00,189 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:01,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:02,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:03,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:04,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:04,775 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:51:05,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:51:05,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:51:05,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:05,208 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:06,192 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:06,943 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:51:06,943 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2e71831cf69946749641.mp4 with policy now
+2025-01-06 22:51:06,945 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:51:06,946 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:06,946 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:07,043 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:07,193 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:07,199 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/ptl12dj7-media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:07,787 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:07,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:07,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:07,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:51:08,071 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/3dnhozdf-media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:08,197 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:09,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:09,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:10,207 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:10,382 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:11,214 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:11,789 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:12,216 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:13,213 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:14,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:15,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:15,790 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:16,131 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:16,211 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:17,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:18,211 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:19,207 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:20,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:51:20,114 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:51:20,210 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:20,502 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:20,502 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:20,505 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:20,562 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:51:20,562 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:51:20,711 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:51:20,792 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:20,792 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:20,792 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:20,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media
+2025-01-06 22:51:20,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table
+2025-01-06 22:51:20,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval
+2025-01-06 22:51:21,033 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/lphlpuz1-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:21,034 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/vnlh6jns-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:21,057 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:51:21,065 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:51:21,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:21,286 INFO    wandb-upload_2:2129923 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7petc0ni
+2025-01-06 22:51:21,774 INFO    SenderThread:2129923 [sender.py:send_request_log_artifact():1525] logged artifact run-bbomq0jb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY0NDUwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2NDEwMQ==', 'latestArtifact': None}}
+2025-01-06 22:51:21,774 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:51:21,774 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json with policy now
+2025-01-06 22:51:21,775 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:21,775 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:21,775 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,775 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,776 DEBUG   SenderThread:2129923 [sender.py:send():379] send: telemetry
+2025-01-06 22:51:21,776 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,779 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,780 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,780 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,780 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,780 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,781 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,781 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,781 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,781 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,782 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,782 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,782 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,782 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,783 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,783 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,783 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,783 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,784 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,784 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,784 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,784 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,785 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval
+2025-01-06 22:51:21,996 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/1k2tyvef-media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:22,024 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/c4or21u1-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:22,141 INFO    wandb-upload_0:2129923 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphp0rib_k
+2025-01-06 22:51:22,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:22,496 INFO    SenderThread:2129923 [sender.py:send_request_log_artifact():1525] logged artifact run-bbomq0jb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY0NDUzMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2NDEwMw==', 'latestArtifact': None}}
+2025-01-06 22:51:22,496 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:51:22,496 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json with policy now
+2025-01-06 22:51:22,497 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:22,497 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:22,497 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:22,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:22,794 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/y7f5u63o-media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:23,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:23,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:24,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:25,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:26,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:27,125 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:27,205 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:27,795 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
+2025-01-06 22:51:28,205 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:29,205 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:29,693 DEBUG   SenderThread:2129923 [sender.py:send():379] send: exit
+2025-01-06 22:51:29,693 INFO    SenderThread:2129923 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 22:51:29,693 INFO    SenderThread:2129923 [sender.py:send_exit():588] handling runtime: 294
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:send_exit():594] send defer
+2025-01-06 22:51:29,694 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,694 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 22:51:29,694 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 1
+2025-01-06 22:51:29,694 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,694 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 22:51:29,695 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,695 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 22:51:29,695 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 2
+2025-01-06 22:51:29,695 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,695 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 22:51:29,695 INFO    HandlerThread:2129923 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 22:51:29,695 DEBUG   SystemMonitor:2129923 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 22:51:29,695 DEBUG   SystemMonitor:2129923 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 22:51:29,696 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 22:51:29,696 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 22:51:29,771 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 22:51:29,771 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 22:51:29,771 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 22:51:29,771 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,771 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 22:51:29,771 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 3
+2025-01-06 22:51:29,771 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:51:29,772 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,772 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 22:51:29,772 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,772 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 22:51:29,772 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 4
+2025-01-06 22:51:29,772 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,772 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 22:51:29,796 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:31,797 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:36,507 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:36,508 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:36,508 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,509 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 22:51:36,509 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 5
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:36,510 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:36,510 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 22:51:36,510 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:36,510 DEBUG   SenderThread:2129923 [sender.py:send():379] send: summary
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:36,511 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 6
+2025-01-06 22:51:36,511 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:36,511 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 22:51:36,511 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 7
+2025-01-06 22:51:36,511 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:36,512 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:36,512 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 22:51:36,512 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,512 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 22:51:36,695 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:36,800 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:38,898 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 8
+2025-01-06 22:51:38,898 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:38,898 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:38,898 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 22:51:38,899 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:38,899 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 22:51:38,899 INFO    SenderThread:2129923 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 22:51:38,901 INFO    SenderThread:2129923 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 22:51:38,903 INFO    SenderThread:2129923 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 22:51:38,917 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 9
+2025-01-06 22:51:38,918 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:38,918 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 22:51:38,918 DEBUG   SenderThread:2129923 [sender.py:send():379] send: artifact
+2025-01-06 22:51:39,117 INFO    SenderThread:2129923 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTc3OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTc3OA==', 'versionIndex': 429}}}
+2025-01-06 22:51:39,117 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:39,117 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 22:51:39,118 INFO    SenderThread:2129923 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 22:51:39,695 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml config.yaml
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0 events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json wandb-summary.json
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt requirements.txt
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 22:51:39,802 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log output.log
+2025-01-06 22:51:39,802 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch diff.patch
+2025-01-06 22:51:39,803 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4 media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4 media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4 media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4 media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4 media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:39,805 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 10
+2025-01-06 22:51:39,806 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:39,807 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:39,807 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 22:51:39,809 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:39,809 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 22:51:39,809 INFO    SenderThread:2129923 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:51:39,995 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:39,997 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
+2025-01-06 22:51:40,018 INFO    wandb-upload_4:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:40,035 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/pli8ax35-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:40,045 INFO    wandb-upload_3:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt
+2025-01-06 22:51:40,245 INFO    Thread-11 (_thread_body):2129923 [sender.py:transition_state():614] send defer: 11
+2025-01-06 22:51:40,245 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,245 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 22:51:40,245 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,245 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 22:51:40,245 INFO    SenderThread:2129923 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 22:51:40,247 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 12
+2025-01-06 22:51:40,247 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,247 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 22:51:40,247 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,247 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 22:51:40,247 INFO    SenderThread:2129923 [file_stream.py:finish():601] file stream finish called
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 13
+2025-01-06 22:51:40,683 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,683 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 22:51:40,683 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 14
+2025-01-06 22:51:40,683 DEBUG   SenderThread:2129923 [sender.py:send():379] send: final
+2025-01-06 22:51:40,684 DEBUG   SenderThread:2129923 [sender.py:send():379] send: footer
+2025-01-06 22:51:40,684 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,684 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 22:51:40,684 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,684 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 22:51:40,684 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:40,684 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:40,685 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 22:51:40,685 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: server_info
+2025-01-06 22:51:40,686 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:40,686 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 22:51:40,687 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 22:51:40,688 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:40,745 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:40,746 INFO    MainThread:2129923 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 22:51:40,746 INFO    MainThread:2129923 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 22:51:40,746 INFO    MainThread:2129923 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 22:51:40,746 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 22:51:40,747 INFO    HandlerThread:2129923 [handler.py:finish():882] shutting down handler
+2025-01-06 22:51:41,687 INFO    WriterThread:2129923 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb
+2025-01-06 22:51:41,746 INFO    SenderThread:2129923 [sender.py:finish():1615] shutting down sender
+2025-01-06 22:51:41,746 INFO    SenderThread:2129923 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:51:41,746 INFO    SenderThread:2129923 [file_pusher.py:join():175] waiting for file pusher
diff --git a/bbomq0jb/wandb/debug.log b/bbomq0jb/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..cea76f03915d0b18fcd6226642702796651c9055
--- /dev/null
+++ b/bbomq0jb/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Configure stats pid to 2129749
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/settings
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug.log
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug-internal.log
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():569] calling init triggers
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb'}
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():619] starting backend
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():623] setting up manager
+2025-01-06 22:46:34,434 INFO    MainThread:2129749 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 22:46:34,434 INFO    MainThread:2129749 [wandb_init.py:init():631] backend started and connected
+2025-01-06 22:46:34,447 INFO    MainThread:2129749 [wandb_init.py:init():720] updated telemetry
+2025-01-06 22:46:34,455 INFO    MainThread:2129749 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 22:46:34,715 INFO    MainThread:2129749 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 22:46:34,758 INFO    MainThread:2129749 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 22:46:34,758 INFO    MainThread:2129749 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 22:46:35,104 INFO    MainThread:2129749 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 22:46:35,105 INFO    MainThread:2129749 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 22:46:35,105 INFO    MainThread:2129749 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 22:46:35,105 INFO    MainThread:2129749 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 22:46:35,106 INFO    MainThread:2129749 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 22:46:35,106 INFO    MainThread:2129749 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/tb, True
+2025-01-06 22:46:39,660 INFO    MainThread:2129749 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2025-01-06 22:51:41,747 WARNING MsgRouterThr:2129749 [router.py:message_loop():77] message_loop has been closed
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cf5de0aa96f303ae2a435a92d9dea98a5569a0d0
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1013_1'
+wandb_notes:
+  desc: null
+  value: 1013_1_restarted_11
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736221594
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: lady
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..65db0437f2055dbb409cbea805d077d23c6305eb
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch
@@ -0,0 +1,63 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index d2afe29..1c0be7a 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1002_0' 
++# wandb_run_id: '1009_1' # seed 1
++# wandb_run_id: '1022_0' # dataset shuffle seed 1
++
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1003_0'
++wandb_run_id: '1013_1' # seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
+-wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1926_1' # seed 1
+ # wandb_run_id: '1930_0' # dataset shuffle seed 1
+ 
+-
+-
+ # RGBD+mask+contact(contact+DTC)+act history
+ # wandb_run_id: '1745_0'
+ 
++# all books, 4/20 demos per book
++
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+ 
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index c7eadce..e3c7dc1 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -161,8 +161,18 @@ run_id_list = [
+ run_id_list = [
+     # '1922_1',
+     # '1922_2',
+-    '1926_1',
+-    '1930_0',
++    # '1944_0',
++    # '1972_0',
++    # '1976_1',
++    # '1948_0',
++    # '1950_0',
++    # '1954_1',
++    '1002_0',
++    '1009_1',
++    '1022_0',
++    '1003_0',
++    '1013_1',
++    '1017_0',
+ ]
+ 
+ checkpoint_epoch = 12000
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..91d1314b0c81ca7ced2bf379f5d1a3bb24013b63
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.20694816118379347, 0.9527384935904063, 0.20744311815390934, 0.9364891498407951, 0.2632499635693074, 0.8492476194398473, 0.24815547471314967, 0.8085234164574604]]}
\ No newline at end of file
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..4b420a8e16940c688624c098ea63200c21c08cec
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 30, 0.8333333333333334, 0.65892112251918, 0.9445055275266461, 0.6608755886232967, 0.9296186267470383, 0.683932982348372, 0.9244472607121343, 0.7465537264450017, 0.9111577644121875]]}
\ No newline at end of file
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4 b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f900f90d764c65723a4be539e3164a43fc3b380b
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a689b02b00d8ffc5d19ed66c1a3d6e28e967f2c362567356590ff20300b76d8
+size 1491278
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4 b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..37229191a873cf7bd7601e48ed7414cd64656f81
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75f06614e77b624267b20ffb66ebd2655692b88759d0376af598c505ff89a322
+size 1484012
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4 b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..b7e9cf7c016a575498c26beea7bb1ddad1f15612
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e12dd92f9893feab8257bff3e23a9443c6ce289e4bf6aead5246edcb4012d1c
+size 1477285
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4 b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..43011bbd0c2404c83f1069614a206ac940ccee16
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9e68fb8bbd70aad7917593a8d03a4b2eec5cc175bbf82ba089da2082f4b21b6
+size 1544991
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4 b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4486e22b47a0f0502a42855db66d58ebdb9ba061
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e71831cf69946749641f10b8512e62d5485ff6d10187e87cc522e0f524e9573
+size 1516284
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..d2743509b2a5508c7cd110924bced9e11b93d513
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
@@ -0,0 +1,272 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736221599.660860]: resetting environment
+[INFO] [1736221599.665163]: cleared current plan
+[INFO] [1736221599.666162]: moving to home
+[INFO] [1736221600.667983]: reached home
+[INFO] [1736221600.669177]: reset action history
+[INFO] [1736221603.379006]: environment reset
+Starting episode 0
+[INFO] [1736221603.383650]: resetting environment
+[INFO] [1736221603.387818]: cleared current plan
+[INFO] [1736221603.388664]: moving to home
+[INFO] [1736221604.391168]: reached home
+[INFO] [1736221604.392526]: reset action history
+[INFO] [1736221607.103798]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736221612.108527]: resetting environment
+[INFO] [1736221612.112921]: cleared current plan
+[INFO] [1736221612.113858]: moving to home
+[INFO] [1736221613.115849]: reached home
+[INFO] [1736221613.117184]: reset action history
+[INFO] [1736221615.827006]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/episode_rosbags/episode_0_2025-01-06-22-46-55.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2129749.log
+started roslaunch server http://158.130.50.37:33525/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2130052]
+process[print_text-2]: started with pid [2130053]
+process[rosbag_record-3]: started with pid [2130054]
+started bagging!
+For topic gripper_width: timestamp difference is 100305231 for nearest: 1736221631957385214 - target: 1736221631857079983 at idx 215
+For topic gripper_width: timestamp difference is 101328176 for nearest: 1736221633257387203 - target: 1736221633156059027 at idx 229
+[INFO] [1736221650.591256]: Storing episode...
+[rosbag_record-3] killing on exit
+[pub_text-1] killing on exit
+[print_text-2] killing on exit
+[INFO] [1736221651.314440]: Stored episode 1.
+[INFO] [1736221651.314719]: Saving video...
+[INFO] [1736221651.505170]: Video saved!
+Starting episode 1
+[INFO] [1736221651.517219]: resetting environment
+[INFO] [1736221651.536553]: cleared current plan
+[INFO] [1736221651.537498]: moving to home
+[INFO] [1736221655.141621]: reached home
+[INFO] [1736221655.142752]: reset action history
+[INFO] [1736221657.854198]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736221668.847728]: resetting environment
+[INFO] [1736221668.852561]: cleared current plan
+[INFO] [1736221668.852729]: moving to home
+[INFO] [1736221669.853273]: reached home
+[INFO] [1736221669.854212]: reset action history
+[INFO] [1736221672.561603]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/episode_rosbags/episode_1_2025-01-06-22-47-52.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2129749.log
+started roslaunch server http://158.130.50.37:33923/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2130179]
+process[print_text-5]: started with pid [2130180]
+process[rosbag_record-6]: started with pid [2130204]
+started bagging!
+For topic gripper_width: timestamp difference is 110102202 for nearest: 1736221685557381417 - target: 1736221685447279215 at idx 161
+[INFO] [1736221707.365120]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736221708.142275]: Stored episode 2.
+[INFO] [1736221708.146648]: Saving video...
+[INFO] [1736221708.632375]: Video saved!
+Starting episode 2
+[INFO] [1736221708.689512]: resetting environment
+[INFO] [1736221708.721207]: cleared current plan
+[INFO] [1736221708.721970]: moving to home
+[WARN] [1736221708.718310]: Plan exhausted
+[INFO] [1736221712.326987]: reached home
+[INFO] [1736221712.331924]: reset action history
+[INFO] [1736221715.111015]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736221726.973499]: resetting environment
+[INFO] [1736221726.988774]: cleared current plan
+[INFO] [1736221726.988967]: moving to home
+[INFO] [1736221727.990493]: reached home
+[INFO] [1736221727.991244]: reset action history
+[INFO] [1736221730.700734]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/episode_rosbags/episode_2_2025-01-06-22-48-50.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2129749.log
+started roslaunch server http://158.130.50.37:40967/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2130348]
+process[print_text-8]: started with pid [2130349]
+process[rosbag_record-9]: started with pid [2130350]
+started bagging!
+For topic gripper_width: timestamp difference is 106320458 for nearest: 1736221763357362586 - target: 1736221763251042128 at idx 249
+For topic gripper_width: timestamp difference is 107383504 for nearest: 1736221764657355800 - target: 1736221764549972296 at idx 249
+[INFO] [1736221765.647761]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1736221766.442581]: Stored episode 3.
+[INFO] [1736221766.448732]: Saving video...
+[INFO] [1736221766.889767]: Video saved!
+Starting episode 3
+[INFO] [1736221766.930369]: resetting environment
+[INFO] [1736221766.944956]: cleared current plan
+[INFO] [1736221766.947416]: moving to home
+[INFO] [1736221770.552544]: reached home
+[INFO] [1736221770.560141]: reset action history
+[INFO] [1736221773.455104]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736221776.367856]: resetting environment
+[INFO] [1736221776.399871]: cleared current plan
+[INFO] [1736221776.407340]: moving to home
+[INFO] [1736221777.417450]: reached home
+[INFO] [1736221777.426105]: reset action history
+[INFO] [1736221780.152289]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/episode_rosbags/episode_3_2025-01-06-22-49-40.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2129749.log
+started roslaunch server http://158.130.50.37:35425/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2130489]
+process[print_text-11]: started with pid [2130490]
+process[rosbag_record-12]: started with pid [2130491]
+started bagging!
+For topic gripper_width: timestamp difference is -118096707 for nearest: 1736221785590713622 - target: 1736221785708810329 at idx 71
+For topic gripper_width: timestamp difference is 104382766 for nearest: 1736221796957382453 - target: 1736221796852999687 at idx 209
+[INFO] [1736221816.146768]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1736221816.894131]: Stored episode 4.
+[INFO] [1736221816.898171]: Saving video...
+[INFO] [1736221817.263225]: Video saved!
+Starting episode 4
+[INFO] [1736221817.322201]: resetting environment
+[INFO] [1736221817.353776]: cleared current plan
+[INFO] [1736221817.357707]: moving to home
+[INFO] [1736221820.767024]: reached home
+[INFO] [1736221820.780065]: reset action history
+[INFO] [1736221823.571383]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736221826.935524]: resetting environment
+[INFO] [1736221826.957531]: cleared current plan
+[INFO] [1736221826.958701]: moving to home
+[INFO] [1736221827.971196]: reached home
+[INFO] [1736221827.974400]: reset action history
+[INFO] [1736221830.711643]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/episode_rosbags/episode_4_2025-01-06-22-50-30.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2129749.log
+started roslaunch server http://158.130.50.37:44229/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2130606]
+process[print_text-14]: started with pid [2130607]
+process[rosbag_record-15]: started with pid [2130631]
+started bagging!
+For topic gripper_width: timestamp difference is 110557830 for nearest: 1736221844457381761 - target: 1736221844346823931 at idx 185
+For topic gripper_width: timestamp difference is 106609075 for nearest: 1736221846257382362 - target: 1736221846150773287 at idx 206
+For topic gripper_width: timestamp difference is 114518604 for nearest: 1736221856157381258 - target: 1736221856042862654 at idx 249
+[INFO] [1736221865.562783]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit[print_text-14] killing on exit
+[INFO] [1736221866.287599]: Stored episode 5.
+[INFO] [1736221866.289587]: Saving video...
+[INFO] [1736221866.887492]: Video saved!
+[WARN] [1736221866.892536]: Plan exhausted
+[WARN] [1736221866.930377]: Plan exhausted
+[INFO] [1736221866.945342]: resetting environment
+[INFO] [1736221866.967738]: cleared current plan
+[INFO] [1736221866.970774]: moving to home
+[INFO] [1736221870.382185]: reached home
+[INFO] [1736221870.388806]: reset action history
+[INFO] [1736221873.171724]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..74dc90d958c4e1f8bdb93ca87d784a25d4a2f8e1
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-07T03:46:34.781844",
+    "startedAt": "2025-01-07T03:46:34.427712",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3937.05225,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 4498.855,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4498.576,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.287,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.998,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.963,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 614.3022308349609
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..9997c82e417a7aefb8d777e0c3962ded3fcd682d
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "4a689b02b00d8ffc5d19ed66c1a3d6e28e967f2c362567356590ff20300b76d8", "size": 1491278, "path": "media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4"}, "global_step": 665, "_timestamp": 1736221880.980791, "_runtime": 286.54569602012634, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "75f06614e77b624267b20ffb66ebd2655692b88759d0376af598c505ff89a322", "size": 1484012, "path": "media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4"}, "eval/num_success": 3.0, "episode": 30.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.2069481611251831, "eval/success_rate_ub_0.01": 0.9527384638786316, "eval/success_rate_lb_0.025": 0.20744311809539795, "eval/success_rate_ub_0.025": 0.9364891648292542, "eval/success_rate_lb_0.05": 0.2632499635219574, "eval/success_rate_ub_0.05": 0.8492476344108582, "eval/success_rate_lb_0.1": 0.24815547466278076, "eval/success_rate_ub_0.1": 0.808523416519165, "eval/2_eval": {"_type": "video-file", "sha256": "2e12dd92f9893feab8257bff3e23a9443c6ce289e4bf6aead5246edcb4012d1c", "size": 1477285, "path": "media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "e9e68fb8bbd70aad7917593a8d03a4b2eec5cc175bbf82ba089da2082f4b21b6", "size": 1544991, "path": "media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "2e71831cf69946749641f10b8512e62d5485ff6d10187e87cc522e0f524e9573", "size": 1516284, "path": "media/videos/eval/4_eval_4_2e71831cf69946749641.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "21a9f3410d4e4ea9386f8f8b05d066553e0d5981bed4f42ec4bc66fd0dcd2c14", "size": 401, "artifact_path": "wandb-client-artifact://dtivh64fyc3h23zjgje7nj9bs7snj2tbrvnwuzbkfi384g5m6s89w5xvusax7wfmldn7gp8004axm85duu6hop0fq9feemhphw2eu152o4d6j12vcz6ts86n6vv6yl2n/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://q0gb8v4b2ohctmbr8vuvsb0zp2koqacok78fbjx19k409s15i6t8tiy6wji6ynqkid2tqpit31es9czez0hopo998pbpr0qxd8dpk3bhnoyt4nz4y5czixs9tipf237k:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 30, "total_success_rate": 0.8333333333333334, "total_success_rate_lb_0.01": 0.65892112251918, "total_success_rate_ub_0.01": 0.9445055275266461, "total_success_rate_lb_0.025": 0.6608755886232967, "total_success_rate_ub_0.025": 0.9296186267470383, "total_success_rate_lb_0.05": 0.683932982348372, "total_success_rate_ub_0.05": 0.9244472607121343, "total_success_rate_lb_0.1": 0.7465537264450017, "total_success_rate_ub_0.1": 0.9111577644121875, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "d97ff5cc0ed67ab5137da62b549a405c3155b53949c369d2098021c59c004b9e", "size": 517, "artifact_path": "wandb-client-artifact://o1rm0jrwuijld48v81pqlgdk0osuo91haypz1o3ezde7zv9guy4shz1inxk9mpag44t5exqy91jegt68m0n7dzsdksgi056f3o1ac21j5vx7ks96xn3ywx0nao6hz871/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://y4rhzktvllmpskt3koybj7ofri1ddcu2qm2u13a24i0iva7cz5u8s4qetuibab719x4op20aofp7suna1c5k7rxzwt43pkj8c410c0sthocaqf5t8vodyj0t4jbzexwp:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 294}, "eval/total_success_rate": 0.8333333134651184, "eval/total_success_rate_lb_0.01": 0.6589211225509644, "eval/total_success_rate_ub_0.01": 0.944505512714386, "eval/total_success_rate_lb_0.025": 0.6608755588531494, "eval/total_success_rate_ub_0.025": 0.9296185970306396, "eval/total_success_rate_lb_0.05": 0.6839329600334167, "eval/total_success_rate_ub_0.05": 0.924447238445282, "eval/total_success_rate_lb_0.1": 0.7465537190437317, "eval/total_success_rate_ub_0.1": 0.9111577868461609}
\ No newline at end of file
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug-internal.log b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..5d2f3003c5eca0240cd1da7a8a511407d1008c4d
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug-internal.log
@@ -0,0 +1,814 @@
+2025-01-06 22:46:34,435 INFO    StreamThr :2129923 [internal.py:wandb_internal():85] W&B internal server running at pid: 2129923, started at: 2025-01-06 22:46:34.434866
+2025-01-06 22:46:34,436 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status
+2025-01-06 22:46:34,447 INFO    WriterThread:2129923 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb
+2025-01-06 22:46:34,448 DEBUG   SenderThread:2129923 [sender.py:send():379] send: header
+2025-01-06 22:46:34,458 DEBUG   SenderThread:2129923 [sender.py:send():379] send: run
+2025-01-06 22:46:34,695 INFO    SenderThread:2129923 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files
+2025-01-06 22:46:34,695 INFO    SenderThread:2129923 [sender.py:_start_run_threads():1188] run started: bbomq0jb with start time 1736221594.435095
+2025-01-06 22:46:34,716 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 22:46:34,716 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: check_version
+2025-01-06 22:46:34,763 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 22:46:34,768 DEBUG   HandlerThread:2129923 [system_info.py:__init__():26] System info init
+2025-01-06 22:46:34,768 DEBUG   HandlerThread:2129923 [system_info.py:__init__():41] System info init done
+2025-01-06 22:46:34,768 INFO    HandlerThread:2129923 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 22:46:34,768 INFO    SystemMonitor:2129923 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 22:46:34,769 INFO    HandlerThread:2129923 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 22:46:34,769 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 22:46:34,770 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 22:46:34,770 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 22:46:34,771 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 22:46:34,771 INFO    SystemMonitor:2129923 [interfaces.py:start():188] Started network monitoring
+2025-01-06 22:46:34,781 DEBUG   HandlerThread:2129923 [system_info.py:probe():152] Probing system
+2025-01-06 22:46:34,783 DEBUG   HandlerThread:2129923 [system_info.py:_probe_git():137] Probing git
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_info.py:probe():200] Probing system done
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:46:34.781844', 'startedAt': '2025-01-07T03:46:34.427712', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.05225, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4498.855, 'min': 800.0, 'max': 4700.0}, {'current': 4498.576, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.287, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 614.3022308349609}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 22:46:34,788 INFO    HandlerThread:2129923 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 22:46:34,788 INFO    HandlerThread:2129923 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 22:46:34,788 DEBUG   HandlerThread:2129923 [system_info.py:_save_code():44] Saving code
+2025-01-06 22:46:34,789 DEBUG   HandlerThread:2129923 [system_info.py:_save_code():67] Saving code done
+2025-01-06 22:46:34,789 DEBUG   HandlerThread:2129923 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 22:46:34,854 DEBUG   HandlerThread:2129923 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 22:46:34,854 INFO    HandlerThread:2129923 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 22:46:34,857 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:46:34,857 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 22:46:34,858 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 22:46:34,858 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 22:46:35,093 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/glhoyajj-wandb-metadata.json
+2025-01-06 22:46:35,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 22:46:35,104 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 22:46:35,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:46:35,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:46:35,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:35,106 INFO    HandlerThread:2129923 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/tb"
+  save: true
+}
+control {
+  relay_id: "cfd43c4364754628a34595dcf6569b62"
+}
+_info {
+  stream_id: "bbomq0jb"
+}
+
+2025-01-06 22:46:35,107 DEBUG   HandlerThread:2129923 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 22:46:35,127 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/mco025zc-diff.patch
+2025-01-06 22:46:35,132 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/6qzbs4ta-code/FISH/eval_robot.py
+2025-01-06 22:46:35,164 DEBUG   SenderThread:2129923 [sender.py:send():379] send: telemetry
+2025-01-06 22:46:35,164 DEBUG   SenderThread:2129923 [sender.py:send():379] send: tbrecord
+2025-01-06 22:46:35,164 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:46:35,164 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0 with policy live
+2025-01-06 22:46:35,335 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/me5c1w3s-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:46:35,697 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json
+2025-01-06 22:46:35,697 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH
+2025-01-06 22:46:35,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code
+2025-01-06 22:46:36,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:37,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:37,696 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:38,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:39,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:39,660 DEBUG   SenderThread:2129923 [sender.py:send():379] send: config
+2025-01-06 22:46:39,660 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:40,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:41,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:41,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:42,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:43,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:43,698 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:44,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:45,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:45,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:45,699 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:46,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:47,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:48,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:49,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:49,700 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:50,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:46:50,104 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:46:50,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:50,196 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:51,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:52,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:53,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:53,701 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:54,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:55,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:55,702 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:55,827 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:46:56,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:57,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:57,703 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:46:58,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:46:59,117 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:00,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:01,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:01,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:02,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:03,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:04,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:05,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:05,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:05,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:06,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:06,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:06,705 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
+2025-01-06 22:47:07,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:08,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:09,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:10,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:11,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:11,960 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:12,116 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:13,117 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:13,707 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:14,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:15,117 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:15,707 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:16,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:17,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:17,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:18,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:19,118 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:20,104 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:20,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:20,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:21,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:22,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:22,304 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:23,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:24,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:25,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:26,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:27,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:28,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:28,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:29,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:30,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:31,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:31,516 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:47:31,516 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4 with policy now
+2025-01-06 22:47:31,516 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:47:31,517 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:47:31,518 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:47:31,518 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos
+2025-01-06 22:47:31,712 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media
+2025-01-06 22:47:31,777 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/0wxi5wtb-media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:47:32,044 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/eeh36p89-media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:47:32,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:33,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:33,538 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:33,713 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:34,120 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:34,771 DEBUG   SystemMonitor:2129923 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 22:47:34,772 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:47:35,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:35,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:35,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:36,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:37,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:37,714 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:38,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:38,901 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:39,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:39,715 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:40,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:41,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:42,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:43,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:44,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:44,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:45,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:46,121 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:47,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:48,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:49,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:49,718 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:47:49,853 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:49,996 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/huzfsfha-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:47:50,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:47:50,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:47:50,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:51,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:51,718 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:52,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:53,122 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:53,719 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:54,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:55,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:47:55,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:55,720 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:47:56,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:57,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:58,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:47:59,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:00,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:00,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:01,123 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:02,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:03,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:04,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:04,772 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:48:05,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:05,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:05,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:05,184 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:06,124 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:07,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:07,724 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:08,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:09,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:10,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:11,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:11,126 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:12,127 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:13,127 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:14,127 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:15,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:16,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:16,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:17,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:18,128 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:19,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:20,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:20,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:20,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:21,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:21,221 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:22,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:23,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:24,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:25,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:26,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:27,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:27,129 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:28,130 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:28,676 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:48:28,676 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4 with policy now
+2025-01-06 22:48:28,679 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:48:28,679 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:48:28,679 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:48:28,684 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:48:28,731 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:48:28,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:48:28,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:48:28,936 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/mgba4a4g-media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:48:29,130 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:29,208 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/o37dwl2w-media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:48:29,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:30,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:30,732 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:31,138 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:32,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:32,140 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:33,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:33,733 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:34,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:34,773 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:48:35,108 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:35,108 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:35,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:36,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:37,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:37,245 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:37,734 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:38,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:39,139 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:40,140 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:41,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:42,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:43,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:43,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:44,142 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:45,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:46,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:47,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:47,540 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:48:47,540 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:48:47,541 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:48:47,738 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:48:47,738 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:48:48,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:49,037 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:49,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:49,738 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:50,105 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:48:50,105 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:48:50,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:51,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:51,739 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:52,143 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:53,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:53,740 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:48:54,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:48:54,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:55,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:56,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:57,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:58,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:48:59,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:00,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:00,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:01,144 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:02,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:03,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:04,145 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:04,773 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:49:05,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:05,106 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:05,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:05,216 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:06,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:07,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:08,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:09,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:10,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:11,111 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:11,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:12,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:13,146 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:14,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:15,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:16,111 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:16,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:17,147 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:18,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:19,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:20,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:20,106 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:20,149 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:21,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:21,195 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:22,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:23,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:24,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:25,148 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:25,751 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:26,150 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:26,445 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:26,922 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:49:26,922 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4 with policy now
+2025-01-06 22:49:26,924 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:49:26,925 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:49:26,925 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:49:26,925 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:49:27,151 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:27,172 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/u2gyat7g-media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:49:27,751 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:49:27,751 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:27,752 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:49:27,752 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:49:28,045 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/xc5pw4o9-media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:49:28,154 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:29,156 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:29,752 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:30,156 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:31,154 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:31,593 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:31,753 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:32,153 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:33,155 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:34,154 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:34,774 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:49:35,110 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:35,110 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:35,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:35,754 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:36,158 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:36,371 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:49:36,371 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:49:36,371 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:49:36,754 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:49:36,755 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:49:36,998 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/12hzf77q-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:49:37,158 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:37,409 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:37,755 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:38,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:39,158 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:39,755 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:40,197 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:41,163 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:41,757 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:42,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:42,696 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:43,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:43,758 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:44,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:45,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:46,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:47,159 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:47,759 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:49:47,833 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:48,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:49,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:50,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:49:50,106 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:49:50,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:51,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:52,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:53,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:53,226 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:54,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:55,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:56,160 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:57,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:58,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:59,009 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:49:59,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:49:59,763 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:00,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:01,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:02,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:03,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:04,113 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:04,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:04,774 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:50:05,106 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:05,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:05,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:06,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:07,161 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:08,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:09,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:09,231 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:10,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:11,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:12,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:13,162 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:14,163 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:15,113 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:15,163 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:16,193 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:17,167 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:17,317 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:50:17,317 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4 with policy now
+2025-01-06 22:50:17,318 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:50:17,319 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:50:17,319 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:50:17,341 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:50:17,582 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/unzssuoz-media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:50:17,769 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:50:17,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:17,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:50:17,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:50:18,038 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/sn86bu0l-media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:50:18,172 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:19,182 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:19,770 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:20,109 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:20,109 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:20,175 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:20,184 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:21,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:21,771 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:22,179 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:23,183 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:24,181 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:25,193 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:25,772 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:26,119 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:26,181 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:27,183 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:27,438 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:50:27,438 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:50:27,438 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:50:27,773 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:27,773 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:50:27,773 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:50:28,183 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:29,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:29,774 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:30,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:31,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:31,774 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:31,983 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:32,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:33,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:33,775 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:34,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:34,775 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:50:35,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:35,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:35,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:36,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:37,184 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:37,185 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:38,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:39,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:40,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:41,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:42,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:43,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:43,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:44,186 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:45,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:45,779 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:46,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:47,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:47,780 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:48,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:48,313 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:49,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:50,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:50:50,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:50:50,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:51,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:52,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:53,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:54,187 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:54,249 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:50:55,188 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:56,213 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:57,188 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:57,783 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:50:58,189 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:50:59,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:00,115 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:00,189 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:01,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:02,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:03,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:04,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:04,775 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:51:05,107 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:51:05,107 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:51:05,190 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:05,208 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:06,192 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:06,943 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:51:06,943 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2e71831cf69946749641.mp4 with policy now
+2025-01-06 22:51:06,945 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:51:06,946 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:06,946 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:07,043 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:07,193 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:07,199 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/ptl12dj7-media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:07,787 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:07,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:07,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:07,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval
+2025-01-06 22:51:08,071 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/3dnhozdf-media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:08,197 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:09,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:09,788 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:10,207 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:10,382 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:11,214 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:11,789 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:12,216 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:13,213 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:14,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:15,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:15,790 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:16,131 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:16,211 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:17,206 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:18,211 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:19,207 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:20,114 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:51:20,114 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:51:20,210 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:20,502 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:20,502 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:20,505 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:20,562 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:51:20,562 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:51:20,711 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:51:20,792 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:20,792 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:20,792 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:20,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media
+2025-01-06 22:51:20,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table
+2025-01-06 22:51:20,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval
+2025-01-06 22:51:21,033 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/lphlpuz1-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:21,034 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/vnlh6jns-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:21,057 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:51:21,065 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:51:21,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:21,286 INFO    wandb-upload_2:2129923 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7petc0ni
+2025-01-06 22:51:21,774 INFO    SenderThread:2129923 [sender.py:send_request_log_artifact():1525] logged artifact run-bbomq0jb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY0NDUwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2NDEwMQ==', 'latestArtifact': None}}
+2025-01-06 22:51:21,774 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:51:21,774 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json with policy now
+2025-01-06 22:51:21,775 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:21,775 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:21,775 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,775 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,776 DEBUG   SenderThread:2129923 [sender.py:send():379] send: telemetry
+2025-01-06 22:51:21,776 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,779 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,780 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,780 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,780 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,780 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,781 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,781 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,781 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,781 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,782 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,782 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,782 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,782 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,783 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,783 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,783 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,783 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,784 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,784 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,784 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:21,784 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:21,785 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:21,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval
+2025-01-06 22:51:21,996 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/1k2tyvef-media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:22,024 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/c4or21u1-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:22,141 INFO    wandb-upload_0:2129923 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphp0rib_k
+2025-01-06 22:51:22,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:22,496 INFO    SenderThread:2129923 [sender.py:send_request_log_artifact():1525] logged artifact run-bbomq0jb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY0NDUzMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2NDEwMw==', 'latestArtifact': None}}
+2025-01-06 22:51:22,496 DEBUG   SenderThread:2129923 [sender.py:send():379] send: files
+2025-01-06 22:51:22,496 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json with policy now
+2025-01-06 22:51:22,497 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:22,497 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:22,497 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:22,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:22,794 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/y7f5u63o-media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:23,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:23,793 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:24,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:25,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:26,204 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:27,125 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:27,205 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:27,795 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
+2025-01-06 22:51:28,205 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:29,205 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:29,693 DEBUG   SenderThread:2129923 [sender.py:send():379] send: exit
+2025-01-06 22:51:29,693 INFO    SenderThread:2129923 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 22:51:29,693 INFO    SenderThread:2129923 [sender.py:send_exit():588] handling runtime: 294
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:send_exit():594] send defer
+2025-01-06 22:51:29,694 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,694 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 22:51:29,694 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 22:51:29,694 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 1
+2025-01-06 22:51:29,694 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,694 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 22:51:29,695 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,695 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 22:51:29,695 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 2
+2025-01-06 22:51:29,695 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,695 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 22:51:29,695 INFO    HandlerThread:2129923 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 22:51:29,695 DEBUG   SystemMonitor:2129923 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 22:51:29,695 DEBUG   SystemMonitor:2129923 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 22:51:29,696 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 22:51:29,696 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 22:51:29,771 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 22:51:29,771 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 22:51:29,771 INFO    HandlerThread:2129923 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 22:51:29,771 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,771 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 22:51:29,771 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 3
+2025-01-06 22:51:29,771 DEBUG   SenderThread:2129923 [sender.py:send():379] send: stats
+2025-01-06 22:51:29,772 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,772 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 22:51:29,772 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:29,772 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 22:51:29,772 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 4
+2025-01-06 22:51:29,772 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:29,772 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 22:51:29,796 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:31,797 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:36,507 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:36,508 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:36,508 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,509 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 22:51:36,509 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 5
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send():379] send: history
+2025-01-06 22:51:36,509 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:51:36,510 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:36,510 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 22:51:36,510 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:36,510 DEBUG   SenderThread:2129923 [sender.py:send():379] send: summary
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:51:36,511 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 6
+2025-01-06 22:51:36,511 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:36,511 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 22:51:36,511 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 22:51:36,511 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 7
+2025-01-06 22:51:36,511 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:51:36,512 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:36,512 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 22:51:36,512 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:36,512 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 22:51:36,695 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:36,800 INFO    Thread-12 :2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:38,898 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 8
+2025-01-06 22:51:38,898 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:38,898 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:38,898 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 22:51:38,899 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:38,899 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 22:51:38,899 INFO    SenderThread:2129923 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 22:51:38,901 INFO    SenderThread:2129923 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 22:51:38,903 INFO    SenderThread:2129923 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 22:51:38,917 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 9
+2025-01-06 22:51:38,918 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:38,918 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 22:51:38,918 DEBUG   SenderThread:2129923 [sender.py:send():379] send: artifact
+2025-01-06 22:51:39,117 INFO    SenderThread:2129923 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTc3OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTc3OA==', 'versionIndex': 429}}}
+2025-01-06 22:51:39,117 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:39,117 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 22:51:39,118 INFO    SenderThread:2129923 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 22:51:39,695 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml config.yaml
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0 events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json wandb-summary.json
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt requirements.txt
+2025-01-06 22:51:39,801 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 22:51:39,802 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log output.log
+2025-01-06 22:51:39,802 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/diff.patch diff.patch
+2025-01-06 22:51:39,803 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/4_eval_4_2e71831cf69946749641.mp4 media/videos/eval/4_eval_4_2e71831cf69946749641.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4 media/videos/eval/1_eval_1_75f06614e77b624267b2.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4 media/videos/eval/3_eval_3_e9e68fb8bbd70aad7917.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4 media/videos/eval/0_eval_0_4a689b02b00d8ffc5d19.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4 media/videos/eval/2_eval_2_2e12dd92f9893feab825.mp4
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json media/table/eval/total_success_rate_ci_6_d97ff5cc0ed67ab5137d.table.json
+2025-01-06 22:51:39,804 INFO    SenderThread:2129923 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json
+2025-01-06 22:51:39,805 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 10
+2025-01-06 22:51:39,806 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:39,807 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:39,807 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 22:51:39,809 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:39,809 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 22:51:39,809 INFO    SenderThread:2129923 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:51:39,995 INFO    wandb-upload_1:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/wandb-summary.json
+2025-01-06 22:51:39,997 INFO    wandb-upload_0:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/config.yaml
+2025-01-06 22:51:40,018 INFO    wandb-upload_4:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/output.log
+2025-01-06 22:51:40,035 INFO    wandb-upload_2:2129923 [upload_job.py:push():130] Uploaded file /tmp/tmpokvkb4jmwandb/pli8ax35-events.out.tfevents.1736221595.leonmkim-ROG-Strix-G15CS-G15CS.2129749.0
+2025-01-06 22:51:40,045 INFO    wandb-upload_3:2129923 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/files/requirements.txt
+2025-01-06 22:51:40,245 INFO    Thread-11 (_thread_body):2129923 [sender.py:transition_state():614] send defer: 11
+2025-01-06 22:51:40,245 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,245 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 22:51:40,245 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,245 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 22:51:40,245 INFO    SenderThread:2129923 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 22:51:40,247 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 12
+2025-01-06 22:51:40,247 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,247 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 22:51:40,247 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,247 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 22:51:40,247 INFO    SenderThread:2129923 [file_stream.py:finish():601] file stream finish called
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 13
+2025-01-06 22:51:40,683 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,683 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 22:51:40,683 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 22:51:40,683 INFO    SenderThread:2129923 [sender.py:transition_state():614] send defer: 14
+2025-01-06 22:51:40,683 DEBUG   SenderThread:2129923 [sender.py:send():379] send: final
+2025-01-06 22:51:40,684 DEBUG   SenderThread:2129923 [sender.py:send():379] send: footer
+2025-01-06 22:51:40,684 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:51:40,684 INFO    HandlerThread:2129923 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 22:51:40,684 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:51:40,684 INFO    SenderThread:2129923 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 22:51:40,684 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:40,684 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:40,685 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 22:51:40,685 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: server_info
+2025-01-06 22:51:40,686 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:51:40,686 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 22:51:40,687 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 22:51:40,688 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:51:40,745 DEBUG   SenderThread:2129923 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:51:40,746 INFO    MainThread:2129923 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 22:51:40,746 INFO    MainThread:2129923 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 22:51:40,746 INFO    MainThread:2129923 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 22:51:40,746 DEBUG   HandlerThread:2129923 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 22:51:40,747 INFO    HandlerThread:2129923 [handler.py:finish():882] shutting down handler
+2025-01-06 22:51:41,687 INFO    WriterThread:2129923 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb
+2025-01-06 22:51:41,746 INFO    SenderThread:2129923 [sender.py:finish():1615] shutting down sender
+2025-01-06 22:51:41,746 INFO    SenderThread:2129923 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:51:41,746 INFO    SenderThread:2129923 [file_pusher.py:join():175] waiting for file pusher
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug.log b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..cea76f03915d0b18fcd6226642702796651c9055
--- /dev/null
+++ b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Configure stats pid to 2129749
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/settings
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 22:46:34,429 INFO    MainThread:2129749 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug.log
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/logs/debug-internal.log
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():569] calling init triggers
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb'}
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():619] starting backend
+2025-01-06 22:46:34,430 INFO    MainThread:2129749 [wandb_init.py:init():623] setting up manager
+2025-01-06 22:46:34,434 INFO    MainThread:2129749 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 22:46:34,434 INFO    MainThread:2129749 [wandb_init.py:init():631] backend started and connected
+2025-01-06 22:46:34,447 INFO    MainThread:2129749 [wandb_init.py:init():720] updated telemetry
+2025-01-06 22:46:34,455 INFO    MainThread:2129749 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 22:46:34,715 INFO    MainThread:2129749 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 22:46:34,758 INFO    MainThread:2129749 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 22:46:34,758 INFO    MainThread:2129749 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 22:46:35,104 INFO    MainThread:2129749 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 22:46:35,105 INFO    MainThread:2129749 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 22:46:35,105 INFO    MainThread:2129749 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 22:46:35,105 INFO    MainThread:2129749 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 22:46:35,106 INFO    MainThread:2129749 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 22:46:35,106 INFO    MainThread:2129749 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bbomq0jb/tb, True
+2025-01-06 22:46:39,660 INFO    MainThread:2129749 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2025-01-06 22:51:41,747 WARNING MsgRouterThr:2129749 [router.py:message_loop():77] message_loop has been closed
diff --git a/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..afe95a208b94b13e2f8cf02d47f812354a82ca3b
Binary files /dev/null and b/bbomq0jb/wandb/run-20250106_224634-bbomq0jb/run-bbomq0jb.wandb differ
diff --git a/no798ka2/episode_rosbags/episode_4_2025-01-07-18-27-12.bag b/no798ka2/episode_rosbags/episode_4_2025-01-07-18-27-12.bag
new file mode 100644
index 0000000000000000000000000000000000000000..f46866520762c62075eb6a4b51e728ad131df850
--- /dev/null
+++ b/no798ka2/episode_rosbags/episode_4_2025-01-07-18-27-12.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1363f6dc51dc1e20868b5eb663941b2cc3b3e1a18839888a15914f3126d8d31a
+size 2604812179
diff --git a/stnsotrb/.hydra/config.yaml b/stnsotrb/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3
--- /dev/null
+++ b/stnsotrb/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1013_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/stnsotrb/.hydra/hydra.yaml b/stnsotrb/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..eb10c0f253a3d064254526465ffb0ae24b4bc059
--- /dev/null
+++ b/stnsotrb/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/stnsotrb
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/stnsotrb/.hydra/overrides.yaml b/stnsotrb/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/stnsotrb/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/stnsotrb/eval_policy.log b/stnsotrb/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..ea54987a380da56d925243ba5b72ed6eacd528b0
--- /dev/null
+++ b/stnsotrb/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-07 20:16:07,951][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 20:16:07,954][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-07 20:16:11,415][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/vhqg7j1c/episode_rosbags/episode_0_2025-01-06-17-55-45.bag b/vhqg7j1c/episode_rosbags/episode_0_2025-01-06-17-55-45.bag
new file mode 100644
index 0000000000000000000000000000000000000000..92a10759d24e79d4e7b5572e6939f43795c8e9b3
--- /dev/null
+++ b/vhqg7j1c/episode_rosbags/episode_0_2025-01-06-17-55-45.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:287c49ec15f00f8e89e7e48e07b5e228da1f46c15337f1b51bf33aab95b30fa9
+size 2621283864
diff --git a/vhqg7j1c/episode_rosbags/episode_2_2025-01-06-17-57-22.bag b/vhqg7j1c/episode_rosbags/episode_2_2025-01-06-17-57-22.bag
new file mode 100644
index 0000000000000000000000000000000000000000..6390090b925f41adcbeed4932efe9b7cc0adb236
--- /dev/null
+++ b/vhqg7j1c/episode_rosbags/episode_2_2025-01-06-17-57-22.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6639afe9222bc88d9378e7e2c558b057843cc73d2b1a05c3634359154c36ba43
+size 2610091030
diff --git a/w4b02c7s/episode_rosbags/episode_3_2025-01-07-20-23-51.bag b/w4b02c7s/episode_rosbags/episode_3_2025-01-07-20-23-51.bag
new file mode 100644
index 0000000000000000000000000000000000000000..f4a3efafc3c131e080edb04d923485fc90c39112
--- /dev/null
+++ b/w4b02c7s/episode_rosbags/episode_3_2025-01-07-20-23-51.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dea384772ae802b128cf45c6f3409f4aa9e41d7371a2f87610e069056a0d580
+size 2596460710
diff --git a/zk6ipw9e/.hydra/config.yaml b/zk6ipw9e/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3
--- /dev/null
+++ b/zk6ipw9e/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1013_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/zk6ipw9e/.hydra/hydra.yaml b/zk6ipw9e/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fb42254858f95fd607f886ab8d5c4403fc236d19
--- /dev/null
+++ b/zk6ipw9e/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/zk6ipw9e/.hydra/overrides.yaml b/zk6ipw9e/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/zk6ipw9e/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/zk6ipw9e/episode_rosbags/aligned_depth_to_color_K.npy b/zk6ipw9e/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/zk6ipw9e/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/zk6ipw9e/episode_rosbags/cam_tf_world.npy b/zk6ipw9e/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/zk6ipw9e/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/zk6ipw9e/episode_rosbags/color_K.npy b/zk6ipw9e/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/zk6ipw9e/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/zk6ipw9e/episode_rosbags/depth_K.npy b/zk6ipw9e/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/zk6ipw9e/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/zk6ipw9e/eval_robot.log b/zk6ipw9e/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..b390c021b587c9d931de3fdea4ade3094bcd9579
--- /dev/null
+++ b/zk6ipw9e/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-07 16:33:56,926][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 16:33:56,929][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/zk6ipw9e/eval_video/0_eval.mp4 b/zk6ipw9e/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..254c054a778ccdaeee8b31be3698ca90cb7f3588
--- /dev/null
+++ b/zk6ipw9e/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ece16d1a03eadbe710f2a457782a3cb658376bf695513c46000320568f06ff
+size 1528527
diff --git a/zk6ipw9e/eval_video/1_eval.mp4 b/zk6ipw9e/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..363756eed3e2800dfcec288777cacb6e21aabc39
--- /dev/null
+++ b/zk6ipw9e/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96ea488e69b84d9427a831dbdcfd5e606310af27efbe40cb6e2f9602de79b0b9
+size 1486490
diff --git a/zk6ipw9e/eval_video/2_eval.mp4 b/zk6ipw9e/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..8cfe3a638a0a48729adf107766ebd966c960314b
--- /dev/null
+++ b/zk6ipw9e/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c977c4b862056286f97dc03cf92b38cefc3602f3eb5612382d19f403e4eda68
+size 1535352
diff --git a/zk6ipw9e/eval_video/3_eval.mp4 b/zk6ipw9e/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..41df78c6463441fe3a223420efb3b6162eca8f27
--- /dev/null
+++ b/zk6ipw9e/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a2bfc48ba7e2cd95e38ed4f48a8c07137a50200e4ea65a8d52c92f8edae2db
+size 1558380
diff --git a/zk6ipw9e/eval_video/4_eval.mp4 b/zk6ipw9e/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2f7f244be3a1ad2e2aab89c81c67518112b158c5
--- /dev/null
+++ b/zk6ipw9e/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576b0ad19b2b7f90da0cb927ce1f6759cfb2133f1ac0698072b4c83be6f97b44
+size 1546408
diff --git a/zk6ipw9e/tb/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0 b/zk6ipw9e/tb/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
new file mode 100644
index 0000000000000000000000000000000000000000..0f02d3d72428895580dab6418cccd9fad444d3e3
--- /dev/null
+++ b/zk6ipw9e/tb/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ca10621ec8345a9e1dc2d507bee012d4ee530d1bb7e4139a5ad0ae3ed0139e
+size 6539
diff --git a/zk6ipw9e/wandb/debug-internal.log b/zk6ipw9e/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..1bc91fd3b48b743f9c54026a7871adebdc08222c
--- /dev/null
+++ b/zk6ipw9e/wandb/debug-internal.log
@@ -0,0 +1,836 @@
+2025-01-07 16:34:02,642 INFO    StreamThr :2186388 [internal.py:wandb_internal():85] W&B internal server running at pid: 2186388, started at: 2025-01-07 16:34:02.642657
+2025-01-07 16:34:02,644 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status
+2025-01-07 16:34:02,657 INFO    WriterThread:2186388 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb
+2025-01-07 16:34:02,657 DEBUG   SenderThread:2186388 [sender.py:send():379] send: header
+2025-01-07 16:34:02,665 DEBUG   SenderThread:2186388 [sender.py:send():379] send: run
+2025-01-07 16:34:02,907 INFO    SenderThread:2186388 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files
+2025-01-07 16:34:02,908 INFO    SenderThread:2186388 [sender.py:_start_run_threads():1188] run started: zk6ipw9e with start time 1736285642.645182
+2025-01-07 16:34:02,923 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: check_version
+2025-01-07 16:34:02,924 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: check_version
+2025-01-07 16:34:02,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: run_start
+2025-01-07 16:34:02,999 DEBUG   HandlerThread:2186388 [system_info.py:__init__():26] System info init
+2025-01-07 16:34:02,999 DEBUG   HandlerThread:2186388 [system_info.py:__init__():41] System info init done
+2025-01-07 16:34:02,999 INFO    HandlerThread:2186388 [system_monitor.py:start():194] Starting system monitor
+2025-01-07 16:34:02,999 INFO    SystemMonitor:2186388 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-07 16:34:02,999 INFO    HandlerThread:2186388 [system_monitor.py:probe():214] Collecting system info
+2025-01-07 16:34:02,999 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started cpu monitoring
+2025-01-07 16:34:03,000 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started disk monitoring
+2025-01-07 16:34:03,000 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started gpu monitoring
+2025-01-07 16:34:03,001 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started memory monitoring
+2025-01-07 16:34:03,002 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started network monitoring
+2025-01-07 16:34:03,012 DEBUG   HandlerThread:2186388 [system_info.py:probe():152] Probing system
+2025-01-07 16:34:03,014 DEBUG   HandlerThread:2186388 [system_info.py:_probe_git():137] Probing git
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_info.py:_probe_git():145] Probing git done
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_info.py:probe():200] Probing system done
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:34:03.012720', 'startedAt': '2025-01-07T21:34:02.634595', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.5906250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.962, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4492.774, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 675.1228561401367}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-07 16:34:03,018 INFO    HandlerThread:2186388 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-07 16:34:03,018 INFO    HandlerThread:2186388 [system_monitor.py:probe():227] Publishing system info
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_info.py:_save_code():44] Saving code
+2025-01-07 16:34:03,020 DEBUG   HandlerThread:2186388 [system_info.py:_save_code():67] Saving code done
+2025-01-07 16:34:03,020 DEBUG   HandlerThread:2186388 [system_info.py:_save_patches():84] Saving git patches
+2025-01-07 16:34:03,082 DEBUG   HandlerThread:2186388 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-07 16:34:03,083 INFO    HandlerThread:2186388 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-07 16:34:03,086 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:34:03,086 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-07 16:34:03,086 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-07 16:34:03,087 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-07 16:34:03,324 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-07 16:34:03,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: python_packages
+2025-01-07 16:34:03,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:03,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:03,326 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:03,348 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/in7artd3-wandb-metadata.json
+2025-01-07 16:34:03,351 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/nqhk9foo-code/FISH/eval_robot.py
+2025-01-07 16:34:03,369 INFO    HandlerThread:2186388 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/tb"
+  save: true
+}
+control {
+  relay_id: "1407d22581d24c6e9b597919d275dcb7"
+}
+_info {
+  stream_id: "zk6ipw9e"
+}
+
+2025-01-07 16:34:03,369 DEBUG   HandlerThread:2186388 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-07 16:34:03,397 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/doye9v5t-diff.patch
+2025-01-07 16:34:03,439 DEBUG   SenderThread:2186388 [sender.py:send():379] send: telemetry
+2025-01-07 16:34:03,439 DEBUG   SenderThread:2186388 [sender.py:send():379] send: tbrecord
+2025-01-07 16:34:03,440 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:34:03,440 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0 with policy live
+2025-01-07 16:34:03,617 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/vfj9y6o7-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:34:03,909 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:03,909 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json
+2025-01-07 16:34:03,909 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH
+2025-01-07 16:34:04,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:05,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:05,908 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:06,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:07,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:07,781 DEBUG   SenderThread:2186388 [sender.py:send():379] send: config
+2025-01-07 16:34:07,781 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:08,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:09,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:09,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:10,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:11,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:12,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:13,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:13,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:13,911 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:14,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:15,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:16,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:17,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:17,912 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:18,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:18,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:18,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:18,474 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:19,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:20,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:21,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:22,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:23,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:23,914 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:24,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:24,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:25,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:25,914 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:26,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:27,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:27,915 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:28,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:29,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:29,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:30,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:31,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:32,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:33,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:33,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:33,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:34,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:34,483 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:34,917 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
+2025-01-07 16:34:35,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:36,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:37,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:38,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:39,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:40,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:40,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:41,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:42,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:43,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:44,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:45,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:45,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:46,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:47,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:48,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:48,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:48,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:49,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:50,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:50,453 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:51,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:52,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:53,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:54,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:55,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:56,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:56,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:57,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:58,337 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:59,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:00,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:01,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:02,026 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:02,238 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:35:02,238 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4 with policy now
+2025-01-07 16:35:02,239 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:35:02,239 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:35:02,240 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:35:02,241 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:35:02,346 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:02,518 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/0mcwhf7l-media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:35:03,002 DEBUG   SystemMonitor:2186388 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-07 16:35:03,002 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:35:03,206 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/r14q5lue-media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:35:03,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:03,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:03,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:03,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:04,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:05,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:06,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:07,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:07,593 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:07,928 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:08,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:09,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:09,929 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:10,350 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:11,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:12,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:13,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:13,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:14,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:15,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:16,350 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:17,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:18,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:18,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:18,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:18,444 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:19,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:20,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:21,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:22,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:23,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:24,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:24,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:24,935 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:35:25,214 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/v7cfmch0-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:35:25,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:25,935 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:26,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:27,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:27,936 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:28,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:29,350 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:29,511 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:29,936 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:30,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:31,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:31,937 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:32,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:33,003 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:35:33,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:33,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:33,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:34,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:35,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:35,458 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:36,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:37,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:38,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:39,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:40,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:41,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:41,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:42,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:43,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:44,354 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:45,354 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:46,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:46,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:47,354 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:48,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:48,327 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:48,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:49,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:50,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:51,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:51,421 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:52,356 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:53,356 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:54,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:55,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:56,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:57,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:57,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:58,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:59,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:00,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:01,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:02,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:03,003 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:36:03,004 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:03,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:03,327 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:03,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:04,360 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:05,282 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:36:05,282 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4 with policy now
+2025-01-07 16:36:05,283 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:36:05,283 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:36:05,283 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:36:05,292 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:36:05,360 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:05,552 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/aqsb5m4l-media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:36:05,948 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:05,948 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:36:05,948 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:36:05,949 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:36:06,226 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/klcme2rr-media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:36:06,361 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:07,364 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:07,949 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:08,296 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:08,368 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:09,366 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:09,949 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:10,364 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:11,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:12,366 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:13,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:13,364 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:13,951 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:14,363 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:14,951 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:36:15,363 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:15,877 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:36:15,877 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:36:15,877 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:36:15,952 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:15,952 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:36:16,370 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:17,370 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:17,952 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:18,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:18,329 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:18,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:18,486 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:19,371 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:19,953 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:20,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:21,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:21,954 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:22,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:23,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:24,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:24,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:25,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:26,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:27,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:28,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:29,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:29,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:30,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:31,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:32,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:33,004 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:36:33,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:33,328 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:33,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:33,957 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:34,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:34,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:35,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:36,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:37,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:37,959 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:38,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:39,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:40,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:40,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:41,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:42,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:43,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:44,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:45,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:46,201 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:46,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:47,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:47,962 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:48,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:48,328 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:48,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:49,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:50,377 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:51,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:51,472 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:52,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:53,376 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:54,376 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:55,037 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:36:55,037 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4 with policy now
+2025-01-07 16:36:55,040 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:36:55,041 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:36:55,041 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:36:55,047 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:36:55,296 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/xx4u3z64-media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:36:55,376 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:36:56,253 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/le8jck9l-media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:36:56,377 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:57,062 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:57,382 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:58,378 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:59,380 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:59,967 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:00,384 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:01,378 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:01,967 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:02,170 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:02,382 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:03,005 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:37:03,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:03,331 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:03,382 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:03,968 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:04,386 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:05,390 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:06,383 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:07,385 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:07,493 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:08,391 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:09,386 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:10,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:11,390 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:12,391 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:13,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:13,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:14,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:14,650 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:37:14,650 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:37:14,650 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:37:14,972 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:37:14,972 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:37:15,222 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/kuvgc5ne-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:37:15,392 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:15,972 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:16,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:17,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:17,973 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:18,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:18,331 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:18,392 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:18,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:19,393 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:19,973 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:20,393 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:21,393 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:21,974 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:22,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:23,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:24,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:24,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:25,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:26,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:27,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:28,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:29,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:29,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:30,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:31,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:32,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:33,005 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:37:33,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:33,331 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:33,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:34,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:34,463 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:35,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:36,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:37,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:38,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:39,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:40,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:40,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:41,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:42,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:43,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:44,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:45,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:45,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:46,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:47,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:48,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:48,332 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:48,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:49,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:50,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:50,444 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:51,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:52,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:53,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:54,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:55,399 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:55,460 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:37:55,461 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4 with policy now
+2025-01-07 16:37:55,461 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:55,505 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:37:55,517 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:37:55,517 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:37:55,523 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:37:55,740 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/7h2c9tl7-media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:37:56,273 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/akftghle-media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:37:56,409 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:57,403 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:57,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:58,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:59,421 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:59,987 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:00,417 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:00,869 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:01,420 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:02,415 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:03,006 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:38:03,340 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:03,340 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:03,418 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:03,988 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:04,419 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:05,420 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:06,425 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:06,497 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:07,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:08,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:09,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:10,431 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:11,430 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:12,345 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:12,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:13,426 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:14,429 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:15,430 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:16,439 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:17,431 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:17,720 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:38:17,720 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:38:17,721 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:17,721 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:38:17,993 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:17,993 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:38:17,993 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:38:18,239 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/eln73211-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:38:18,341 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:18,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:18,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:19,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:19,994 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:20,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:21,431 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:21,994 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:22,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:23,338 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:23,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:23,995 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:24,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:25,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:26,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:27,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:28,339 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:28,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:29,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:30,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:31,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:32,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:33,007 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:38:33,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:33,334 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:33,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:33,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:34,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:35,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:36,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:37,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:38,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:39,339 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:39,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:40,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:41,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:42,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:43,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:44,339 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:44,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:45,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:46,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:47,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:48,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:48,333 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:48,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:49,427 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:49,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:50,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:51,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:52,005 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:52,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:53,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:54,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:55,340 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:55,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:56,437 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:57,443 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:57,998 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:38:57,998 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4 with policy now
+2025-01-07 16:38:58,005 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:38:58,006 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:38:58,006 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:38:58,006 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:38:58,007 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:58,007 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:38:58,007 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:38:58,008 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:38:58,261 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/0qyugqoi-media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:38:58,438 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:58,551 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/tojh0gyr-media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:38:59,442 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:00,008 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:00,341 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:00,447 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:01,445 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:02,446 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:03,007 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:39:03,341 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:39:03,342 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:39:03,452 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:04,009 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:04,454 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:05,451 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:06,010 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:06,225 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:06,447 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:07,461 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:08,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:09,457 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:10,465 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:10,786 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:10,786 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:10,787 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:11,011 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:11,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:11,441 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 16:39:11,442 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 16:39:11,466 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:11,621 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:39:11,917 INFO    wandb-upload_0:2186388 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkphipljx
+2025-01-07 16:39:12,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:12,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:12,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:12,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media
+2025-01-07 16:39:12,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table
+2025-01-07 16:39:12,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval
+2025-01-07 16:39:12,049 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 16:39:12,058 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:39:12,330 INFO    SenderThread:2186388 [sender.py:send_request_log_artifact():1525] logged artifact run-zk6ipw9e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MTkwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY2MTc1OQ==', 'latestArtifact': None}}
+2025-01-07 16:39:12,331 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:39:12,331 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now
+2025-01-07 16:39:12,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:12,331 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:12,332 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,332 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,333 DEBUG   SenderThread:2186388 [sender.py:send():379] send: telemetry
+2025-01-07 16:39:12,333 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,337 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,337 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,337 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,337 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,338 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,338 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,338 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,338 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,339 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,339 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,339 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,339 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,340 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,340 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,340 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,340 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,340 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,341 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,341 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 16:39:12,461 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:12,523 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/9dvm67kc-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:12,737 INFO    wandb-upload_0:2186388 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp845_3_z8
+2025-01-07 16:39:12,767 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/kvgw6mow-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:13,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:13,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:13,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval
+2025-01-07 16:39:13,116 INFO    SenderThread:2186388 [sender.py:send_request_log_artifact():1525] logged artifact run-zk6ipw9e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MTkyNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY2MTc2Mw==', 'latestArtifact': None}}
+2025-01-07 16:39:13,116 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:39:13,116 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json with policy now
+2025-01-07 16:39:13,117 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:13,117 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:13,117 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:13,332 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/mrinnl8c-media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:13,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:13,569 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/hpbzdcdn-media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:14,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:14,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:14,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:15,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:15,993 DEBUG   SenderThread:2186388 [sender.py:send():379] send: exit
+2025-01-07 16:39:15,993 INFO    SenderThread:2186388 [sender.py:send_exit():586] handling exit code: 0
+2025-01-07 16:39:15,993 INFO    SenderThread:2186388 [sender.py:send_exit():588] handling runtime: 312
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:send_exit():594] send defer
+2025-01-07 16:39:15,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:15,994 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-07 16:39:15,994 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 1
+2025-01-07 16:39:15,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:15,994 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-07 16:39:15,995 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:15,995 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-07 16:39:15,995 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 2
+2025-01-07 16:39:15,995 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:15,995 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-07 16:39:15,995 INFO    HandlerThread:2186388 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-07 16:39:15,995 DEBUG   SystemMonitor:2186388 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-07 16:39:15,996 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-07 16:39:15,996 DEBUG   SystemMonitor:2186388 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-07 16:39:15,996 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined disk monitor
+2025-01-07 16:39:16,014 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:16,050 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-07 16:39:16,050 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined memory monitor
+2025-01-07 16:39:16,050 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined network monitor
+2025-01-07 16:39:16,051 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 3
+2025-01-07 16:39:16,051 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:39:16,051 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:16,051 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-07 16:39:16,051 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 4
+2025-01-07 16:39:16,051 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:16,052 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-07 16:39:18,015 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:22,793 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:22,793 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:22,794 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,796 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-07 16:39:22,796 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 5
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:22,797 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:22,797 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-07 16:39:22,797 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:22,798 DEBUG   SenderThread:2186388 [sender.py:send():379] send: summary
+2025-01-07 16:39:22,798 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:22,798 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,798 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-07 16:39:22,798 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 6
+2025-01-07 16:39:22,798 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:22,798 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-07 16:39:22,798 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,799 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-07 16:39:22,802 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:22,912 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 7
+2025-01-07 16:39:22,912 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:22,912 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-07 16:39:22,912 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,912 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-07 16:39:22,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:23,017 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:23,017 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
+2025-01-07 16:39:23,755 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 8
+2025-01-07 16:39:23,755 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:23,755 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:23,755 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-07 16:39:23,756 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:23,756 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-07 16:39:23,756 INFO    SenderThread:2186388 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-07 16:39:23,758 INFO    SenderThread:2186388 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-07 16:39:23,760 INFO    SenderThread:2186388 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-07 16:39:23,772 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 9
+2025-01-07 16:39:23,772 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:23,772 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-07 16:39:23,773 DEBUG   SenderThread:2186388 [sender.py:send():379] send: artifact
+2025-01-07 16:39:23,995 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:24,018 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:24,558 INFO    wandb-upload_0:2186388 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj785suu_
+2025-01-07 16:39:24,559 INFO    wandb-upload_1:2186388 [upload_job.py:push():85] Skipped uploading /tmp/tmp80dn15u1/wandb-job.json
+2025-01-07 16:39:24,710 INFO    wandb-upload_2:2186388 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgbdqwmw3
+2025-01-07 16:39:25,342 INFO    SenderThread:2186388 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'versionIndex': 431}}}
+2025-01-07 16:39:25,342 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:25,342 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-07 16:39:25,342 INFO    SenderThread:2186388 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml config.yaml
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json wandb-summary.json
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt requirements.txt
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json wandb-metadata.json
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0 events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log output.log
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch diff.patch
+2025-01-07 16:39:26,020 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4 media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4 media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4 media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4 media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4 media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:39:26,026 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:26,027 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:26,028 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 10
+2025-01-07 16:39:26,028 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:26,028 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,029 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-07 16:39:26,029 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,029 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-07 16:39:26,029 INFO    SenderThread:2186388 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 16:39:26,199 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
+2025-01-07 16:39:26,205 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:26,241 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt
+2025-01-07 16:39:26,259 INFO    wandb-upload_4:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:26,344 INFO    wandb-upload_3:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/s0o38i0c-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:26,544 INFO    Thread-11 (_thread_body):2186388 [sender.py:transition_state():614] send defer: 11
+2025-01-07 16:39:26,544 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,544 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-07 16:39:26,545 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,545 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-07 16:39:26,545 INFO    SenderThread:2186388 [file_pusher.py:join():175] waiting for file pusher
+2025-01-07 16:39:26,546 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 12
+2025-01-07 16:39:26,546 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,546 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-07 16:39:26,546 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,546 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-07 16:39:26,546 INFO    SenderThread:2186388 [file_stream.py:finish():601] file stream finish called
+2025-01-07 16:39:26,723 INFO    SenderThread:2186388 [file_stream.py:finish():605] file stream finish is done
+2025-01-07 16:39:26,723 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 13
+2025-01-07 16:39:26,723 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,723 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,724 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-07 16:39:26,724 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 14
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send():379] send: final
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send():379] send: footer
+2025-01-07 16:39:26,724 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,724 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,724 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: server_info
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-07 16:39:26,726 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:26,726 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-07 16:39:26,726 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: server_info
+2025-01-07 16:39:26,727 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:26,778 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:26,778 INFO    MainThread:2186388 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-07 16:39:26,779 INFO    MainThread:2186388 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-07 16:39:26,779 INFO    MainThread:2186388 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-07 16:39:26,779 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-07 16:39:26,779 INFO    HandlerThread:2186388 [handler.py:finish():882] shutting down handler
+2025-01-07 16:39:27,725 INFO    WriterThread:2186388 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb
+2025-01-07 16:39:27,778 INFO    SenderThread:2186388 [sender.py:finish():1615] shutting down sender
+2025-01-07 16:39:27,778 INFO    SenderThread:2186388 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 16:39:27,778 INFO    SenderThread:2186388 [file_pusher.py:join():175] waiting for file pusher
diff --git a/zk6ipw9e/wandb/debug.log b/zk6ipw9e/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..1ca2b378a89919a0f8e32ba904c6eb35265d7c6f
--- /dev/null
+++ b/zk6ipw9e/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Configure stats pid to 2186195
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/settings
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug.log
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug-internal.log
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_init.py:init():569] calling init triggers
+2025-01-07 16:34:02,638 INFO    MainThread:2186195 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e'}
+2025-01-07 16:34:02,638 INFO    MainThread:2186195 [wandb_init.py:init():619] starting backend
+2025-01-07 16:34:02,638 INFO    MainThread:2186195 [wandb_init.py:init():623] setting up manager
+2025-01-07 16:34:02,641 INFO    MainThread:2186195 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-07 16:34:02,644 INFO    MainThread:2186195 [wandb_init.py:init():631] backend started and connected
+2025-01-07 16:34:02,657 INFO    MainThread:2186195 [wandb_init.py:init():720] updated telemetry
+2025-01-07 16:34:02,663 INFO    MainThread:2186195 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-07 16:34:02,923 INFO    MainThread:2186195 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-07 16:34:02,988 INFO    MainThread:2186195 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-07 16:34:02,989 INFO    MainThread:2186195 [wandb_init.py:init():804] starting run threads in backend
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-07 16:34:03,327 INFO    MainThread:2186195 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-07 16:34:03,328 INFO    MainThread:2186195 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/tb, True
+2025-01-07 16:34:07,780 INFO    MainThread:2186195 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-07 16:39:27,780 WARNING MsgRouterThr:2186195 [router.py:message_loop():77] message_loop has been closed
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..61220b29199e56bda3a2971ba19de94d0e9656da
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1013_1'
+wandb_notes:
+  desc: null
+  value: 1013_1_restarted_11
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736285642
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..8adda0290b36965f42f4119e05fd13ea9507158f
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch
@@ -0,0 +1,15 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index a5e82a4..1c0be7a 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -139,8 +139,8 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1003_0'
+-# wandb_run_id: '1013_1' # seed 1
+-wandb_run_id: '1017_0' # dataset shuffle seed 1
++wandb_run_id: '1013_1' # seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]}
\ No newline at end of file
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..567bd8bfa7ace16167b2148c928177de8d970cb2
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 35, 0.7142857142857143, 0.535616815083056, 0.8592796771999871, 0.5435139238747355, 0.837847106091164, 0.5661325827108679, 0.8275408967674348, 0.6212690733130628, 0.8084057382622458]]}
\ No newline at end of file
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4 b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..254c054a778ccdaeee8b31be3698ca90cb7f3588
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ece16d1a03eadbe710f2a457782a3cb658376bf695513c46000320568f06ff
+size 1528527
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4 b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..363756eed3e2800dfcec288777cacb6e21aabc39
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96ea488e69b84d9427a831dbdcfd5e606310af27efbe40cb6e2f9602de79b0b9
+size 1486490
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4 b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..8cfe3a638a0a48729adf107766ebd966c960314b
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c977c4b862056286f97dc03cf92b38cefc3602f3eb5612382d19f403e4eda68
+size 1535352
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4 b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..41df78c6463441fe3a223420efb3b6162eca8f27
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a2bfc48ba7e2cd95e38ed4f48a8c07137a50200e4ea65a8d52c92f8edae2db
+size 1558380
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4 b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2f7f244be3a1ad2e2aab89c81c67518112b158c5
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576b0ad19b2b7f90da0cb927ce1f6759cfb2133f1ac0698072b4c83be6f97b44
+size 1546408
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..3fe9101cae9da678ff63d87577624b5ad0933dd8
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
@@ -0,0 +1,266 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736285647.781478]: resetting environment
+[INFO] [1736285647.788013]: cleared current plan
+[INFO] [1736285647.789121]: moving to home
+[INFO] [1736285648.790752]: reached home
+[INFO] [1736285648.792212]: reset action history
+[INFO] [1736285651.502490]: environment reset
+Starting episode 0
+[INFO] [1736285651.511537]: resetting environment
+[INFO] [1736285651.518967]: cleared current plan
+[INFO] [1736285651.519865]: moving to home
+[INFO] [1736285652.521399]: reached home
+[INFO] [1736285652.522578]: reset action history
+[INFO] [1736285655.232449]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1736285662.244949]: resetting environment
+[INFO] [1736285662.251294]: cleared current plan
+[INFO] [1736285662.252546]: moving to home
+[INFO] [1736285663.254609]: reached home
+[INFO] [1736285663.255900]: reset action history
+[INFO] [1736285665.963230]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/episode_rosbags/episode_0_2025-01-07-16-34-25.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2186195.log
+started roslaunch server http://158.130.50.37:43573/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2186521]
+process[print_text-2]: started with pid [2186522]
+process[rosbag_record-3]: started with pid [2186523]
+started bagging!
+[INFO] [1736285701.267894]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1736285702.026314]: Stored episode 1.
+[INFO] [1736285702.026558]: Saving video...
+[INFO] [1736285702.224149]: Video saved!
+Starting episode 1
+[INFO] [1736285702.240371]: resetting environment
+[INFO] [1736285702.247932]: cleared current plan
+[INFO] [1736285702.248532]: moving to home
+[INFO] [1736285705.552013]: reached home
+[INFO] [1736285705.552441]: reset action history
+[INFO] [1736285708.314261]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736285724.558265]: resetting environment
+[INFO] [1736285724.576054]: cleared current plan
+[INFO] [1736285724.576671]: moving to home
+[INFO] [1736285725.577351]: reached home
+[INFO] [1736285725.577827]: reset action history
+[INFO] [1736285728.293658]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/episode_rosbags/episode_1_2025-01-07-16-35-28.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2186195.log
+started roslaunch server http://158.130.50.37:33671/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2186669]
+process[print_text-5]: started with pid [2186670]
+process[rosbag_record-6]: started with pid [2186671]
+started bagging!
+[INFO] [1736285764.238806]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736285765.024465]: Stored episode 2.
+[INFO] [1736285765.035629]: Saving video...
+[INFO] [1736285765.251009]: Video saved!
+Starting episode 2
+[INFO] [1736285765.284748]: resetting environment
+[INFO] [1736285765.295169]: cleared current plan
+[INFO] [1736285765.295412]: moving to home
+[INFO] [1736285768.599970]: reached home
+[INFO] [1736285768.603812]: reset action history
+[INFO] [1736285771.329319]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736285774.854578]: resetting environment
+[INFO] [1736285774.876369]: cleared current plan
+[INFO] [1736285774.878901]: moving to home
+[INFO] [1736285775.884777]: reached home
+[INFO] [1736285775.885305]: reset action history
+[INFO] [1736285778.612028]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/episode_rosbags/episode_2_2025-01-07-16-36-18.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2186195.log
+started roslaunch server http://158.130.50.37:46537/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2186812]
+process[print_text-8]: started with pid [2186813]
+process[rosbag_record-9]: started with pid [2186814]
+started bagging!
+For topic gripper_width: timestamp difference is 103185573 for nearest: 1736285792445535579 - target: 1736285792342350006 at idx 183
+For topic gripper_width: timestamp difference is 105757355 for nearest: 1736285795545535206 - target: 1736285795439777851 at idx 230
+For topic gripper_width: timestamp difference is 111282534 for nearest: 1736285805145533747 - target: 1736285805034251213 at idx 249
+[INFO] [1736285813.952533]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1736285814.774830]: Stored episode 3.
+[INFO] [1736285814.778581]: Saving video...
+[INFO] [1736285815.008201]: Video saved!
+Starting episode 3
+[INFO] [1736285815.041839]: resetting environment
+[INFO] [1736285815.061402]: cleared current plan
+[INFO] [1736285815.062083]: moving to home
+[INFO] [1736285818.165511]: reached home
+[INFO] [1736285818.169441]: reset action history
+[INFO] [1736285821.123887]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736285834.601540]: resetting environment
+[INFO] [1736285834.624228]: cleared current plan
+[INFO] [1736285834.626179]: moving to home
+[INFO] [1736285835.630208]: reached home
+[INFO] [1736285835.633235]: reset action history
+[INFO] [1736285838.376002]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/episode_rosbags/episode_3_2025-01-07-16-37-18.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2186195.log
+started roslaunch server http://158.130.50.37:45141/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2186957]
+process[print_text-11]: started with pid [2186958]
+process[rosbag_record-12]: started with pid [2186959]
+started bagging!
+For topic gripper_width: timestamp difference is 101551801 for nearest: 1736285840545532495 - target: 1736285840443980694 at idx 37
+[INFO] [1736285874.391266]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1736285875.134836]: Stored episode 4.
+[INFO] [1736285875.140368]: Saving video...
+[INFO] [1736285875.385490]: Video saved!
+Starting episode 4
+[INFO] [1736285875.467576]: resetting environment
+[INFO] [1736285875.497540]: cleared current plan
+[INFO] [1736285875.516904]: moving to home
+[INFO] [1736285878.825322]: reached home
+[INFO] [1736285878.829456]: reset action history
+[INFO] [1736285881.558468]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736285897.250788]: resetting environment
+[INFO] [1736285897.273729]: cleared current plan
+[INFO] [1736285897.277003]: moving to home
+[INFO] [1736285898.283856]: reached home
+[INFO] [1736285898.286975]: reset action history
+[INFO] [1736285901.066590]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/episode_rosbags/episode_4_2025-01-07-16-38-21.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2186195.log
+started roslaunch server http://158.130.50.37:37943/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2187105]
+process[print_text-14]: started with pid [2187106]
+process[rosbag_record-15]: started with pid [2187107]
+started bagging!
+For topic gripper_width: timestamp difference is 119680923 for nearest: 1736285930945531887 - target: 1736285930825850964 at idx 249
+[INFO] [1736285936.928320]: Storing episode...
+[pub_text-13] killing on exit
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[INFO] [1736285937.724728]: Stored episode 5.
+[INFO] [1736285937.729654]: Saving video...
+[INFO] [1736285937.955549]: Video saved!
+[INFO] [1736285938.007394]: resetting environment
+[INFO] [1736285938.020808]: cleared current plan
+[INFO] [1736285938.021052]: moving to home
+[INFO] [1736285941.323980]: reached home
+[INFO] [1736285941.332277]: reset action history
+[INFO] [1736285944.181834]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea7be452f7fec9ce425783040e21385d966a556e
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-07T21:34:03.012720",
+    "startedAt": "2025-01-07T21:34:02.634595",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "29237eae507484d4aff7683bad40a564046255b9"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3561.5906250000003,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 4499.962,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4492.774,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 675.1228561401367
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d2d55c651ee57685025263ebe3fa0f48dcab97d
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "22ece16d1a03eadbe710f2a457782a3cb658376bf695513c46000320568f06ff", "size": 1528527, "path": "media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4"}, "global_step": 665, "_timestamp": 1736285951.9423482, "_runtime": 309.29716634750366, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "96ea488e69b84d9427a831dbdcfd5e606310af27efbe40cb6e2f9602de79b0b9", "size": 1486490, "path": "media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4"}, "eval/num_success": 0.0, "episode": 35.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "8c977c4b862056286f97dc03cf92b38cefc3602f3eb5612382d19f403e4eda68", "size": 1535352, "path": "media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f4a2bfc48ba7e2cd95e38ed4f48a8c07137a50200e4ea65a8d52c92f8edae2db", "size": 1558380, "path": "media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "576b0ad19b2b7f90da0cb927ce1f6759cfb2133f1ac0698072b4c83be6f97b44", "size": 1546408, "path": "media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://20gy18xbp6v6qc0ulwx789372ysjbtjc6zm9dbwc3tq27sdud0fanik3kq21hxr5n7zvu4dluhmm0grucbumf4hv02ekx6po06v8lv4nwuv09if6fhifgejtve2mshku/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://u3ux44psddk20mi0fasq764ky9ovdi8p6z57si34mv4ez0t17wdf4ojp7e052fya1gi0hivkrogmf0wuc458cxzdl77y2pln0ravno7igq94lqvpm6f53te4gclphv95:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 35, "total_success_rate": 0.7142857142857143, "total_success_rate_lb_0.01": 0.535616815083056, "total_success_rate_ub_0.01": 0.8592796771999871, "total_success_rate_lb_0.025": 0.5435139238747355, "total_success_rate_ub_0.025": 0.837847106091164, "total_success_rate_lb_0.05": 0.5661325827108679, "total_success_rate_ub_0.05": 0.8275408967674348, "total_success_rate_lb_0.1": 0.6212690733130628, "total_success_rate_ub_0.1": 0.8084057382622458, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "947eacd30b5aa61b138be4144a5fd3bfceeed19cdc538c21e9457943dcd943e1", "size": 518, "artifact_path": "wandb-client-artifact://xdz5n4pz3qt64d9pni0aaowh4m1g9mlrjg665lo0rc73stu678pwwqqkllp6qd6uts38g2oy434lgrccla1z145o4efcmsqmjv8gs9igblrwvi0aunurubjcxdtbugv4/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://c355lc8cqw7rn6oqjg6aux5c31twk2fre95994zda3xtutu3b5i6uabszfx86gud7hup4amigmgut034h7w8ne7p89hmktg2597lnoy2xvt0lyo67oxfm3xkfslo8026:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 312}, "eval/total_success_rate": 0.7142857313156128, "eval/total_success_rate_lb_0.01": 0.5356168150901794, "eval/total_success_rate_ub_0.01": 0.8592796921730042, "eval/total_success_rate_lb_0.025": 0.5435138940811157, "eval/total_success_rate_ub_0.025": 0.837847113609314, "eval/total_success_rate_lb_0.05": 0.5661326050758362, "eval/total_success_rate_ub_0.05": 0.8275408744812012, "eval/total_success_rate_lb_0.1": 0.6212690472602844, "eval/total_success_rate_ub_0.1": 0.8084057569503784}
\ No newline at end of file
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug-internal.log b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..1bc91fd3b48b743f9c54026a7871adebdc08222c
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug-internal.log
@@ -0,0 +1,836 @@
+2025-01-07 16:34:02,642 INFO    StreamThr :2186388 [internal.py:wandb_internal():85] W&B internal server running at pid: 2186388, started at: 2025-01-07 16:34:02.642657
+2025-01-07 16:34:02,644 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status
+2025-01-07 16:34:02,657 INFO    WriterThread:2186388 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb
+2025-01-07 16:34:02,657 DEBUG   SenderThread:2186388 [sender.py:send():379] send: header
+2025-01-07 16:34:02,665 DEBUG   SenderThread:2186388 [sender.py:send():379] send: run
+2025-01-07 16:34:02,907 INFO    SenderThread:2186388 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files
+2025-01-07 16:34:02,908 INFO    SenderThread:2186388 [sender.py:_start_run_threads():1188] run started: zk6ipw9e with start time 1736285642.645182
+2025-01-07 16:34:02,923 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: check_version
+2025-01-07 16:34:02,924 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: check_version
+2025-01-07 16:34:02,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: run_start
+2025-01-07 16:34:02,999 DEBUG   HandlerThread:2186388 [system_info.py:__init__():26] System info init
+2025-01-07 16:34:02,999 DEBUG   HandlerThread:2186388 [system_info.py:__init__():41] System info init done
+2025-01-07 16:34:02,999 INFO    HandlerThread:2186388 [system_monitor.py:start():194] Starting system monitor
+2025-01-07 16:34:02,999 INFO    SystemMonitor:2186388 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-07 16:34:02,999 INFO    HandlerThread:2186388 [system_monitor.py:probe():214] Collecting system info
+2025-01-07 16:34:02,999 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started cpu monitoring
+2025-01-07 16:34:03,000 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started disk monitoring
+2025-01-07 16:34:03,000 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started gpu monitoring
+2025-01-07 16:34:03,001 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started memory monitoring
+2025-01-07 16:34:03,002 INFO    SystemMonitor:2186388 [interfaces.py:start():188] Started network monitoring
+2025-01-07 16:34:03,012 DEBUG   HandlerThread:2186388 [system_info.py:probe():152] Probing system
+2025-01-07 16:34:03,014 DEBUG   HandlerThread:2186388 [system_info.py:_probe_git():137] Probing git
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_info.py:_probe_git():145] Probing git done
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_info.py:probe():200] Probing system done
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:34:03.012720', 'startedAt': '2025-01-07T21:34:02.634595', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.5906250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.962, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4492.774, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 675.1228561401367}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-07 16:34:03,018 INFO    HandlerThread:2186388 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-07 16:34:03,018 INFO    HandlerThread:2186388 [system_monitor.py:probe():227] Publishing system info
+2025-01-07 16:34:03,018 DEBUG   HandlerThread:2186388 [system_info.py:_save_code():44] Saving code
+2025-01-07 16:34:03,020 DEBUG   HandlerThread:2186388 [system_info.py:_save_code():67] Saving code done
+2025-01-07 16:34:03,020 DEBUG   HandlerThread:2186388 [system_info.py:_save_patches():84] Saving git patches
+2025-01-07 16:34:03,082 DEBUG   HandlerThread:2186388 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-07 16:34:03,083 INFO    HandlerThread:2186388 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-07 16:34:03,086 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:34:03,086 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-07 16:34:03,086 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-07 16:34:03,087 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-07 16:34:03,324 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-07 16:34:03,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: python_packages
+2025-01-07 16:34:03,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:03,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:03,326 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:03,348 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/in7artd3-wandb-metadata.json
+2025-01-07 16:34:03,351 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/nqhk9foo-code/FISH/eval_robot.py
+2025-01-07 16:34:03,369 INFO    HandlerThread:2186388 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/tb"
+  save: true
+}
+control {
+  relay_id: "1407d22581d24c6e9b597919d275dcb7"
+}
+_info {
+  stream_id: "zk6ipw9e"
+}
+
+2025-01-07 16:34:03,369 DEBUG   HandlerThread:2186388 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-07 16:34:03,397 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/doye9v5t-diff.patch
+2025-01-07 16:34:03,439 DEBUG   SenderThread:2186388 [sender.py:send():379] send: telemetry
+2025-01-07 16:34:03,439 DEBUG   SenderThread:2186388 [sender.py:send():379] send: tbrecord
+2025-01-07 16:34:03,440 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:34:03,440 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0 with policy live
+2025-01-07 16:34:03,617 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/vfj9y6o7-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:34:03,909 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:03,909 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json
+2025-01-07 16:34:03,909 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code
+2025-01-07 16:34:03,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH
+2025-01-07 16:34:04,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:05,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:05,908 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:06,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:07,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:07,781 DEBUG   SenderThread:2186388 [sender.py:send():379] send: config
+2025-01-07 16:34:07,781 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:08,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:09,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:09,910 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:10,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:11,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:12,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:13,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:13,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:13,911 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:14,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:15,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:16,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:17,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:17,912 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:18,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:18,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:18,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:18,474 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:19,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:20,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:21,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:22,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:23,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:23,914 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:24,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:24,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:25,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:25,914 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:26,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:27,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:27,915 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:34:28,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:29,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:29,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:30,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:31,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:32,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:33,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:33,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:33,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:34,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:34,483 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:34,917 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
+2025-01-07 16:34:35,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:36,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:37,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:38,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:39,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:40,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:40,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:41,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:42,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:43,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:44,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:45,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:45,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:46,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:47,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:48,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:34:48,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:34:48,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:49,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:50,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:50,453 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:51,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:52,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:53,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:54,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:55,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:56,327 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:34:56,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:57,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:58,337 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:34:59,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:00,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:01,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:02,026 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:02,238 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:35:02,238 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4 with policy now
+2025-01-07 16:35:02,239 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:35:02,239 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:35:02,240 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:35:02,241 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:35:02,346 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:02,518 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/0mcwhf7l-media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media
+2025-01-07 16:35:02,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:35:03,002 DEBUG   SystemMonitor:2186388 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-07 16:35:03,002 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:35:03,206 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/r14q5lue-media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:35:03,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:03,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:03,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:03,926 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:04,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:05,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:06,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:07,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:07,593 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:07,928 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:08,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:09,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:09,929 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:10,350 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:11,347 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:12,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:13,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:13,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:14,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:15,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:16,350 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:17,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:18,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:18,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:18,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:18,444 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:19,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:20,348 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:21,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:22,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:23,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:24,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:24,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:24,935 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:35:25,214 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/v7cfmch0-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:35:25,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:25,935 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:26,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:27,349 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:27,936 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:28,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:29,350 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:29,511 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:29,936 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:30,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:31,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:31,937 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:35:32,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:33,003 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:35:33,325 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:33,325 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:33,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:34,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:35,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:35,458 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:36,352 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:37,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:38,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:39,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:40,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:41,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:41,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:42,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:43,353 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:44,354 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:45,354 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:46,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:46,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:47,354 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:48,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:35:48,327 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:35:48,369 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:49,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:50,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:51,355 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:51,421 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:52,356 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:53,356 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:54,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:55,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:56,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:57,329 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:35:57,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:58,357 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:35:59,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:00,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:01,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:02,358 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:03,003 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:36:03,004 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:03,326 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:03,327 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:03,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:04,360 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:05,282 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:36:05,282 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4 with policy now
+2025-01-07 16:36:05,283 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:36:05,283 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:36:05,283 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:36:05,292 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:36:05,360 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:05,552 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/aqsb5m4l-media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:36:05,948 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:05,948 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:36:05,948 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:36:05,949 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:36:06,226 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/klcme2rr-media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:36:06,361 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:07,364 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:07,949 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:08,296 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:08,368 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:09,366 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:09,949 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:10,364 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:11,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:12,366 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:13,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:13,364 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:13,951 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:14,363 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:14,951 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:36:15,363 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:15,877 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:36:15,877 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:36:15,877 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:36:15,952 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:15,952 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:36:16,370 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:17,370 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:17,952 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:18,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:18,329 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:18,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:18,486 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:19,371 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:19,953 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:20,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:21,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:21,954 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:22,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:23,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:24,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:24,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:25,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:26,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:27,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:28,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:29,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:29,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:30,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:31,372 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:32,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:33,004 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:36:33,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:33,328 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:33,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:33,957 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:34,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:34,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:35,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:36,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:37,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:37,959 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:38,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:39,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:40,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:40,373 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:41,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:42,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:43,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:44,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:45,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:46,201 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:46,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:47,374 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:47,962 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:48,328 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:36:48,328 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:36:48,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:49,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:50,377 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:51,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:51,472 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:52,375 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:53,376 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:54,376 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:55,037 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:36:55,037 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4 with policy now
+2025-01-07 16:36:55,040 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:36:55,041 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:36:55,041 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:36:55,047 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:36:55,296 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/xx4u3z64-media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:36:55,376 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:36:55,965 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:36:56,253 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/le8jck9l-media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:36:56,377 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:57,062 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:36:57,382 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:58,378 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:59,380 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:36:59,967 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:00,384 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:01,378 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:01,967 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:02,170 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:02,382 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:03,005 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:37:03,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:03,331 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:03,382 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:03,968 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:04,386 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:05,390 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:06,383 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:07,385 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:07,493 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:08,391 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:09,386 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:10,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:11,390 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:12,391 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:13,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:13,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:14,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:14,650 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:37:14,650 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:37:14,650 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:37:14,972 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:37:14,972 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:37:15,222 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/kuvgc5ne-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:37:15,392 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:15,972 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:16,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:17,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:17,973 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:18,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:18,331 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:18,392 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:18,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:19,393 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:19,973 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:20,393 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:21,393 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:21,974 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:22,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:23,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:24,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:24,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:25,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:26,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:27,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:28,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:29,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:29,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:30,394 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:31,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:32,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:33,005 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:37:33,330 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:33,331 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:33,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:34,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:34,463 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:35,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:36,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:37,395 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:38,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:39,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:40,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:40,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:41,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:42,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:43,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:44,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:45,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:45,396 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:46,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:47,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:48,332 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:37:48,332 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:37:48,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:49,397 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:50,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:50,444 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:51,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:52,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:53,398 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:54,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:55,399 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:55,460 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:37:55,461 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4 with policy now
+2025-01-07 16:37:55,461 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:37:55,505 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:37:55,517 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:37:55,517 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:37:55,523 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:37:55,740 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/7h2c9tl7-media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:37:55,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:37:56,273 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/akftghle-media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:37:56,409 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:57,403 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:57,986 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:37:58,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:59,421 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:37:59,987 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:00,417 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:00,869 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:01,420 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:02,415 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:03,006 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:38:03,340 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:03,340 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:03,418 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:03,988 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:04,419 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:05,420 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:06,425 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:06,497 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:07,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:08,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:09,428 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:10,431 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:11,430 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:12,345 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:12,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:13,426 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:14,429 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:15,430 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:16,439 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:17,431 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:17,720 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:38:17,720 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:38:17,721 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:17,721 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:38:17,993 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:17,993 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:38:17,993 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:38:18,239 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/eln73211-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:38:18,341 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:18,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:18,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:19,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:19,994 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:20,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:21,431 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:21,994 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:22,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:23,338 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:23,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:23,995 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:24,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:25,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:26,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:27,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:28,339 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:28,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:29,432 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:30,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:31,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:32,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:33,007 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:38:33,334 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:33,334 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:33,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:33,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:34,433 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:35,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:36,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:37,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:38,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:39,339 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:39,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:40,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:41,434 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:42,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:43,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:44,339 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:44,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:45,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:46,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:47,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:48,333 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:38:48,333 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:38:48,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:49,427 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:49,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:50,435 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:51,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:52,005 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:52,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:53,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:54,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:55,340 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:38:55,436 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:56,437 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:57,443 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:57,998 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:38:57,998 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4 with policy now
+2025-01-07 16:38:58,005 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:38:58,006 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:38:58,006 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:38:58,006 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:38:58,007 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:38:58,007 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:38:58,007 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:38:58,008 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval
+2025-01-07 16:38:58,261 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/0qyugqoi-media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:38:58,438 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:38:58,551 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/tojh0gyr-media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:38:59,442 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:00,008 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:00,341 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:00,447 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:01,445 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:02,446 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:03,007 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:39:03,341 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 16:39:03,342 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 16:39:03,452 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:04,009 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:04,454 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:05,451 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:06,010 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:06,225 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:06,447 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:07,461 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:08,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:09,457 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:10,465 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:10,786 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:10,786 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:10,787 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:11,011 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:11,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:11,441 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 16:39:11,442 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 16:39:11,466 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:11,621 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:39:11,917 INFO    wandb-upload_0:2186388 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkphipljx
+2025-01-07 16:39:12,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:12,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:12,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:12,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media
+2025-01-07 16:39:12,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table
+2025-01-07 16:39:12,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval
+2025-01-07 16:39:12,049 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 16:39:12,058 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 16:39:12,330 INFO    SenderThread:2186388 [sender.py:send_request_log_artifact():1525] logged artifact run-zk6ipw9e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MTkwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY2MTc1OQ==', 'latestArtifact': None}}
+2025-01-07 16:39:12,331 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:39:12,331 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now
+2025-01-07 16:39:12,331 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:12,331 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:12,332 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,332 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,333 DEBUG   SenderThread:2186388 [sender.py:send():379] send: telemetry
+2025-01-07 16:39:12,333 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,337 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,337 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,337 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,337 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,338 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,338 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,338 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,338 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,339 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,339 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,339 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,339 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,340 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,340 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,340 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,340 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,340 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,341 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:12,341 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:12,341 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 16:39:12,461 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:12,523 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/9dvm67kc-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:12,737 INFO    wandb-upload_0:2186388 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp845_3_z8
+2025-01-07 16:39:12,767 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/kvgw6mow-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:13,012 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:13,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:13,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval
+2025-01-07 16:39:13,116 INFO    SenderThread:2186388 [sender.py:send_request_log_artifact():1525] logged artifact run-zk6ipw9e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MTkyNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY2MTc2Mw==', 'latestArtifact': None}}
+2025-01-07 16:39:13,116 DEBUG   SenderThread:2186388 [sender.py:send():379] send: files
+2025-01-07 16:39:13,116 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json with policy now
+2025-01-07 16:39:13,117 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:13,117 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:13,117 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:13,332 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/mrinnl8c-media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:13,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:13,569 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/hpbzdcdn-media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:14,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:14,013 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:14,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:15,462 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:15,993 DEBUG   SenderThread:2186388 [sender.py:send():379] send: exit
+2025-01-07 16:39:15,993 INFO    SenderThread:2186388 [sender.py:send_exit():586] handling exit code: 0
+2025-01-07 16:39:15,993 INFO    SenderThread:2186388 [sender.py:send_exit():588] handling runtime: 312
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:send_exit():594] send defer
+2025-01-07 16:39:15,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:15,994 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-07 16:39:15,994 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-07 16:39:15,994 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 1
+2025-01-07 16:39:15,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:15,994 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-07 16:39:15,995 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:15,995 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-07 16:39:15,995 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 2
+2025-01-07 16:39:15,995 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:15,995 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-07 16:39:15,995 INFO    HandlerThread:2186388 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-07 16:39:15,995 DEBUG   SystemMonitor:2186388 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-07 16:39:15,996 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-07 16:39:15,996 DEBUG   SystemMonitor:2186388 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-07 16:39:15,996 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined disk monitor
+2025-01-07 16:39:16,014 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:16,050 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-07 16:39:16,050 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined memory monitor
+2025-01-07 16:39:16,050 INFO    HandlerThread:2186388 [interfaces.py:finish():200] Joined network monitor
+2025-01-07 16:39:16,051 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 3
+2025-01-07 16:39:16,051 DEBUG   SenderThread:2186388 [sender.py:send():379] send: stats
+2025-01-07 16:39:16,051 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:16,051 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-07 16:39:16,051 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-07 16:39:16,051 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 4
+2025-01-07 16:39:16,051 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:16,052 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-07 16:39:18,015 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:22,793 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:22,793 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:22,794 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,796 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-07 16:39:22,796 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 5
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send():379] send: history
+2025-01-07 16:39:22,796 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 16:39:22,797 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:22,797 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-07 16:39:22,797 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:22,798 DEBUG   SenderThread:2186388 [sender.py:send():379] send: summary
+2025-01-07 16:39:22,798 INFO    SenderThread:2186388 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 16:39:22,798 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,798 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-07 16:39:22,798 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 6
+2025-01-07 16:39:22,798 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:22,798 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-07 16:39:22,798 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,799 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-07 16:39:22,802 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 16:39:22,912 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 7
+2025-01-07 16:39:22,912 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:22,912 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-07 16:39:22,912 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:22,912 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-07 16:39:22,994 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:23,017 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:23,017 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
+2025-01-07 16:39:23,755 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 8
+2025-01-07 16:39:23,755 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:23,755 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:23,755 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-07 16:39:23,756 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:23,756 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-07 16:39:23,756 INFO    SenderThread:2186388 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-07 16:39:23,758 INFO    SenderThread:2186388 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-07 16:39:23,760 INFO    SenderThread:2186388 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-07 16:39:23,772 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 9
+2025-01-07 16:39:23,772 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:23,772 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-07 16:39:23,773 DEBUG   SenderThread:2186388 [sender.py:send():379] send: artifact
+2025-01-07 16:39:23,995 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:24,018 INFO    Thread-12 :2186388 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:24,558 INFO    wandb-upload_0:2186388 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj785suu_
+2025-01-07 16:39:24,559 INFO    wandb-upload_1:2186388 [upload_job.py:push():85] Skipped uploading /tmp/tmp80dn15u1/wandb-job.json
+2025-01-07 16:39:24,710 INFO    wandb-upload_2:2186388 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgbdqwmw3
+2025-01-07 16:39:25,342 INFO    SenderThread:2186388 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'versionIndex': 431}}}
+2025-01-07 16:39:25,342 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:25,342 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-07 16:39:25,342 INFO    SenderThread:2186388 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml config.yaml
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json wandb-summary.json
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt requirements.txt
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-metadata.json wandb-metadata.json
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0 events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log output.log
+2025-01-07 16:39:26,019 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/diff.patch diff.patch
+2025-01-07 16:39:26,020 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4 media/videos/eval/1_eval_1_96ea488e69b84d9427a8.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4 media/videos/eval/3_eval_3_f4a2bfc48ba7e2cd95e3.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4 media/videos/eval/2_eval_2_8c977c4b862056286f97.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4 media/videos/eval/0_eval_0_22ece16d1a03eadbe710.mp4
+2025-01-07 16:39:26,025 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4 media/videos/eval/4_eval_4_576b0ad19b2b7f90da0c.mp4
+2025-01-07 16:39:26,026 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json
+2025-01-07 16:39:26,027 INFO    SenderThread:2186388 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json media/table/eval/total_success_rate_ci_6_947eacd30b5aa61b138b.table.json
+2025-01-07 16:39:26,028 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 10
+2025-01-07 16:39:26,028 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:26,028 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,029 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-07 16:39:26,029 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,029 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-07 16:39:26,029 INFO    SenderThread:2186388 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 16:39:26,199 INFO    wandb-upload_0:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/config.yaml
+2025-01-07 16:39:26,205 INFO    wandb-upload_1:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/wandb-summary.json
+2025-01-07 16:39:26,241 INFO    wandb-upload_2:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/requirements.txt
+2025-01-07 16:39:26,259 INFO    wandb-upload_4:2186388 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/files/output.log
+2025-01-07 16:39:26,344 INFO    wandb-upload_3:2186388 [upload_job.py:push():130] Uploaded file /tmp/tmp4mukrqjqwandb/s0o38i0c-events.out.tfevents.1736285643.leonmkim-ROG-Strix-G15CS-G15CS.2186195.0
+2025-01-07 16:39:26,544 INFO    Thread-11 (_thread_body):2186388 [sender.py:transition_state():614] send defer: 11
+2025-01-07 16:39:26,544 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,544 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-07 16:39:26,545 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,545 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-07 16:39:26,545 INFO    SenderThread:2186388 [file_pusher.py:join():175] waiting for file pusher
+2025-01-07 16:39:26,546 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 12
+2025-01-07 16:39:26,546 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,546 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-07 16:39:26,546 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,546 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-07 16:39:26,546 INFO    SenderThread:2186388 [file_stream.py:finish():601] file stream finish called
+2025-01-07 16:39:26,723 INFO    SenderThread:2186388 [file_stream.py:finish():605] file stream finish is done
+2025-01-07 16:39:26,723 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 13
+2025-01-07 16:39:26,723 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,723 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,724 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-07 16:39:26,724 INFO    SenderThread:2186388 [sender.py:transition_state():614] send defer: 14
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send():379] send: final
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send():379] send: footer
+2025-01-07 16:39:26,724 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 16:39:26,724 INFO    HandlerThread:2186388 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-07 16:39:26,724 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: defer
+2025-01-07 16:39:26,724 INFO    SenderThread:2186388 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: server_info
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 16:39:26,725 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-07 16:39:26,726 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:26,726 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-07 16:39:26,726 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: server_info
+2025-01-07 16:39:26,727 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 16:39:26,778 DEBUG   SenderThread:2186388 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 16:39:26,778 INFO    MainThread:2186388 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-07 16:39:26,779 INFO    MainThread:2186388 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-07 16:39:26,779 INFO    MainThread:2186388 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-07 16:39:26,779 DEBUG   HandlerThread:2186388 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-07 16:39:26,779 INFO    HandlerThread:2186388 [handler.py:finish():882] shutting down handler
+2025-01-07 16:39:27,725 INFO    WriterThread:2186388 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb
+2025-01-07 16:39:27,778 INFO    SenderThread:2186388 [sender.py:finish():1615] shutting down sender
+2025-01-07 16:39:27,778 INFO    SenderThread:2186388 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 16:39:27,778 INFO    SenderThread:2186388 [file_pusher.py:join():175] waiting for file pusher
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug.log b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..1ca2b378a89919a0f8e32ba904c6eb35265d7c6f
--- /dev/null
+++ b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Configure stats pid to 2186195
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/settings
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug.log
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/logs/debug-internal.log
+2025-01-07 16:34:02,637 INFO    MainThread:2186195 [wandb_init.py:init():569] calling init triggers
+2025-01-07 16:34:02,638 INFO    MainThread:2186195 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e'}
+2025-01-07 16:34:02,638 INFO    MainThread:2186195 [wandb_init.py:init():619] starting backend
+2025-01-07 16:34:02,638 INFO    MainThread:2186195 [wandb_init.py:init():623] setting up manager
+2025-01-07 16:34:02,641 INFO    MainThread:2186195 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-07 16:34:02,644 INFO    MainThread:2186195 [wandb_init.py:init():631] backend started and connected
+2025-01-07 16:34:02,657 INFO    MainThread:2186195 [wandb_init.py:init():720] updated telemetry
+2025-01-07 16:34:02,663 INFO    MainThread:2186195 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-07 16:34:02,923 INFO    MainThread:2186195 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-07 16:34:02,988 INFO    MainThread:2186195 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-07 16:34:02,989 INFO    MainThread:2186195 [wandb_init.py:init():804] starting run threads in backend
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-07 16:34:03,326 INFO    MainThread:2186195 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-07 16:34:03,327 INFO    MainThread:2186195 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-07 16:34:03,328 INFO    MainThread:2186195 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/zk6ipw9e/tb, True
+2025-01-07 16:34:07,780 INFO    MainThread:2186195 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-07 16:39:27,780 WARNING MsgRouterThr:2186195 [router.py:message_loop():77] message_loop has been closed
diff --git a/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..cc0f39dae85d21edc43c3e27e4ff22649b8cbde8
Binary files /dev/null and b/zk6ipw9e/wandb/run-20250107_163402-zk6ipw9e/run-zk6ipw9e.wandb differ