diff --git a/.gitattributes b/.gitattributes
index d8c6e6d2b35e7e2c8cbcaf6769dfff05e22a6c4d..d563358042ab0e4fe440c602afc4b7629b750842 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -179,3 +179,80 @@ q1ti958r/episode_rosbags/episode_10_2025-01-18-18-57-52.bag filter=lfs diff=lfs
 q1ti958r/episode_rosbags/episode_0_2025-01-18-18-50-38.bag filter=lfs diff=lfs merge=lfs -text
 q1ti958r/episode_rosbags/episode_11_2025-01-18-18-58-35.bag filter=lfs diff=lfs merge=lfs -text
 q1ti958r/episode_rosbags/episode_19_2025-01-18-19-05-54.bag filter=lfs diff=lfs merge=lfs -text
+mo0b6mh9/episode_rosbags/episode_3_2025-01-18-19-57-53.bag filter=lfs diff=lfs merge=lfs -text
+mo0b6mh9/episode_rosbags/episode_7_2025-01-18-20-00-52.bag filter=lfs diff=lfs merge=lfs -text
+mo0b6mh9/episode_rosbags/episode_6_2025-01-18-20-00-08.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_9_2025-01-18-18-57-05.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_2_2025-01-18-18-51-59.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4 filter=lfs diff=lfs merge=lfs -text
+mo0b6mh9/episode_rosbags/episode_19_2025-01-18-20-11-01.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_3_2025-01-18-18-52-46.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_16_2025-01-18-19-02-54.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_5_2025-01-18-18-54-09.bag filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_18_2025-01-18-19-05-03.bag filter=lfs diff=lfs merge=lfs -text
+rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_4_2025-01-18-18-53-26.bag filter=lfs diff=lfs merge=lfs -text
+rgws6h04/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+q1ti958r/episode_rosbags/episode_12_2025-01-18-18-59-17.bag filter=lfs diff=lfs merge=lfs -text
+rgws6h04/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+mo0b6mh9/episode_rosbags/episode_1_2025-01-18-19-56-25.bag filter=lfs diff=lfs merge=lfs -text
+138tidw7/episode_rosbags/episode_9_2025-01-18-19-38-32.bag filter=lfs diff=lfs merge=lfs -text
+mo0b6mh9/episode_rosbags/episode_14_2025-01-18-20-06-53.bag filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4 filter=lfs diff=lfs merge=lfs -text
+rgws6h04/episode_rosbags/episode_0_2025-01-21-20-46-39.bag filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+66rp0oj9/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
diff --git a/138tidw7/episode_rosbags/episode_9_2025-01-18-19-38-32.bag b/138tidw7/episode_rosbags/episode_9_2025-01-18-19-38-32.bag
new file mode 100644
index 0000000000000000000000000000000000000000..aa2f41924079b90bba3eaff24c7787ba994c92dc
--- /dev/null
+++ b/138tidw7/episode_rosbags/episode_9_2025-01-18-19-38-32.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0874c6d39d1cc692f9d76977d3a4878f5abad1e76eb6a0c2a2c86e21dc216234
+size 1414124228
diff --git a/66rp0oj9/.hydra/config.yaml b/66rp0oj9/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6f829134d57914397f938447dc055c8c1721578e
--- /dev/null
+++ b/66rp0oj9/.hydra/config.yaml
@@ -0,0 +1,352 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '3465_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 7500
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      crop_distractors_tlhw: null
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+      crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw}
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/66rp0oj9/.hydra/hydra.yaml b/66rp0oj9/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5b30b8c0d4493d7d1c5daff05f19c8d03ca0b0ad
--- /dev/null
+++ b/66rp0oj9/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/66rp0oj9/.hydra/overrides.yaml b/66rp0oj9/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/66rp0oj9/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/66rp0oj9/episode_rosbags/aligned_depth_to_color_K.npy b/66rp0oj9/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/66rp0oj9/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/66rp0oj9/episode_rosbags/cam_tf_world.npy b/66rp0oj9/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/66rp0oj9/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/66rp0oj9/episode_rosbags/color_K.npy b/66rp0oj9/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/66rp0oj9/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/66rp0oj9/episode_rosbags/depth_K.npy b/66rp0oj9/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/66rp0oj9/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/66rp0oj9/eval_robot.log b/66rp0oj9/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..06f5b88ca08ceda4a5a1543f3e0e792ec652052f
--- /dev/null
+++ b/66rp0oj9/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-21 20:51:38,386][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-21 20:51:38,389][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/66rp0oj9/eval_video/0_eval.mp4 b/66rp0oj9/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bbf13b25e8ba984cd867973e1150a8575cb1239c
--- /dev/null
+++ b/66rp0oj9/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10f2d5a1476b38f259e0bd9786e10425512ac938efc84e83d6a0cf7e00e4d860
+size 1173896
diff --git a/66rp0oj9/eval_video/1_eval.mp4 b/66rp0oj9/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4973064d55e977d23ed7fed749b82d358bf795e6
--- /dev/null
+++ b/66rp0oj9/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d14dcbccf52d3b425c83b0b193b4d4335542e2a2655a13a02e11d4b72394c121
+size 1173762
diff --git a/66rp0oj9/eval_video/2_eval.mp4 b/66rp0oj9/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4f05d4c40bd71d49a2f1324c0431a3a7196fc3c3
--- /dev/null
+++ b/66rp0oj9/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:660acc7709e365474234422a1637eca0b03396ba0dd76d4a713e46bd3597a440
+size 1188354
diff --git a/66rp0oj9/eval_video/3_eval.mp4 b/66rp0oj9/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ebf893f14c735464376fdf3401be6e9154b39e4f
--- /dev/null
+++ b/66rp0oj9/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf508a8824a44b8c6adbba807899817a3119200de11972a46b32b83dc94d868d
+size 1182542
diff --git a/66rp0oj9/eval_video/4_eval.mp4 b/66rp0oj9/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..502dd62bb842ec7b17464d890b95b5d7307ce1f7
--- /dev/null
+++ b/66rp0oj9/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75f1e1972486326d53ed64a7ae2ba2f5db26aa29381c7d76bc21f2944f5f06be
+size 1237160
diff --git a/66rp0oj9/tb/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0 b/66rp0oj9/tb/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
new file mode 100644
index 0000000000000000000000000000000000000000..b8579f3c2d1f950d25938edba681536eecca9004
--- /dev/null
+++ b/66rp0oj9/tb/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dc9bd73093e5d30f8787e7c4b7a34e9ee19979d6b650f3a8545e05ecb9eec7e
+size 6539
diff --git a/66rp0oj9/wandb/debug-internal.log b/66rp0oj9/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c8944505778f13666ee039b8730e7431cda6f7e7
--- /dev/null
+++ b/66rp0oj9/wandb/debug-internal.log
@@ -0,0 +1,729 @@
+2025-01-21 20:51:44,292 INFO    StreamThr :2641882 [internal.py:wandb_internal():85] W&B internal server running at pid: 2641882, started at: 2025-01-21 20:51:44.292377
+2025-01-21 20:51:44,293 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status
+2025-01-21 20:51:44,304 INFO    WriterThread:2641882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb
+2025-01-21 20:51:44,305 DEBUG   SenderThread:2641882 [sender.py:send():379] send: header
+2025-01-21 20:51:44,315 DEBUG   SenderThread:2641882 [sender.py:send():379] send: run
+2025-01-21 20:51:44,594 INFO    SenderThread:2641882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files
+2025-01-21 20:51:44,594 INFO    SenderThread:2641882 [sender.py:_start_run_threads():1188] run started: 66rp0oj9 with start time 1737510704.292423
+2025-01-21 20:51:44,611 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: check_version
+2025-01-21 20:51:44,611 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: check_version
+2025-01-21 20:51:44,654 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: run_start
+2025-01-21 20:51:44,659 DEBUG   HandlerThread:2641882 [system_info.py:__init__():26] System info init
+2025-01-21 20:51:44,659 DEBUG   HandlerThread:2641882 [system_info.py:__init__():41] System info init done
+2025-01-21 20:51:44,659 INFO    HandlerThread:2641882 [system_monitor.py:start():194] Starting system monitor
+2025-01-21 20:51:44,659 INFO    SystemMonitor:2641882 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-21 20:51:44,659 INFO    HandlerThread:2641882 [system_monitor.py:probe():214] Collecting system info
+2025-01-21 20:51:44,660 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started cpu monitoring
+2025-01-21 20:51:44,661 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started disk monitoring
+2025-01-21 20:51:44,661 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started gpu monitoring
+2025-01-21 20:51:44,662 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started memory monitoring
+2025-01-21 20:51:44,662 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started network monitoring
+2025-01-21 20:51:44,677 DEBUG   HandlerThread:2641882 [system_info.py:probe():152] Probing system
+2025-01-21 20:51:44,679 DEBUG   HandlerThread:2641882 [system_info.py:_probe_git():137] Probing git
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_info.py:_probe_git():145] Probing git done
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_info.py:probe():200] Probing system done
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:51:44.677333', 'startedAt': '2025-01-22T01:51:44.284707', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4122.9805, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.584, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 322.3823585510254}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-21 20:51:44,684 INFO    HandlerThread:2641882 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-21 20:51:44,684 INFO    HandlerThread:2641882 [system_monitor.py:probe():227] Publishing system info
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_info.py:_save_code():44] Saving code
+2025-01-21 20:51:44,686 DEBUG   HandlerThread:2641882 [system_info.py:_save_code():67] Saving code done
+2025-01-21 20:51:44,686 DEBUG   HandlerThread:2641882 [system_info.py:_save_patches():84] Saving git patches
+2025-01-21 20:51:44,747 DEBUG   HandlerThread:2641882 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-21 20:51:44,747 INFO    HandlerThread:2641882 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-21 20:51:44,750 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:51:44,750 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-21 20:51:44,751 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-21 20:51:44,751 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-21 20:51:44,986 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-21 20:51:44,986 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: python_packages
+2025-01-21 20:51:44,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:51:44,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:51:44,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:44,990 INFO    HandlerThread:2641882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/tb"
+  save: true
+}
+control {
+  relay_id: "12c704b22f184c97abe0deaf0658d3b2"
+}
+_info {
+  stream_id: "66rp0oj9"
+}
+
+2025-01-21 20:51:44,990 DEBUG   HandlerThread:2641882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-21 20:51:45,020 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/j27huxi2-wandb-metadata.json
+2025-01-21 20:51:45,052 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/6mgydkap-diff.patch
+2025-01-21 20:51:45,062 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/ksqjbenx-code/FISH/eval_robot.py
+2025-01-21 20:51:45,069 DEBUG   SenderThread:2641882 [sender.py:send():379] send: telemetry
+2025-01-21 20:51:45,069 DEBUG   SenderThread:2641882 [sender.py:send():379] send: tbrecord
+2025-01-21 20:51:45,070 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:51:45,070 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0 with policy live
+2025-01-21 20:51:45,295 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/6wiua7gr-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:51:45,601 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch
+2025-01-21 20:51:45,601 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json
+2025-01-21 20:51:45,601 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH
+2025-01-21 20:51:45,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:46,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:47,600 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:47,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:48,616 DEBUG   SenderThread:2641882 [sender.py:send():379] send: config
+2025-01-21 20:51:48,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:49,623 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:51:49,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:50,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:51,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:51,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:52,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:53,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:53,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:54,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:51:54,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:55,603 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:55,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:56,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:57,603 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:57,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:58,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:59,604 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:59,986 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:51:59,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:00,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:00,080 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:00,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:01,605 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:01,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:02,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:03,605 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:03,990 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:05,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:05,990 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:06,034 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:06,990 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:07,607 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:07,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:08,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:09,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:10,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:11,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:11,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:12,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:13,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:14,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:14,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:15,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:15,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:16,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:17,128 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:17,609 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
+2025-01-21 20:52:17,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:18,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:19,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:20,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:21,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:22,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:22,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:23,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:24,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:25,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:26,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:27,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:27,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:28,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:29,320 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:52:29,320 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4 with policy now
+2025-01-21 20:52:29,320 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:52:29,321 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:52:29,322 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:52:29,322 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:52:29,587 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/qp87anz1-media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:52:29,612 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:52:29,906 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/3mze40wd-media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:52:29,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:29,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:29,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:30,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:31,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:31,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:32,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:33,985 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:33,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:34,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:35,614 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:35,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:36,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:37,615 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:37,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:38,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:39,616 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:52:39,913 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:39,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:40,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:41,616 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:41,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:42,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:43,617 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:43,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:44,663 DEBUG   SystemMonitor:2641882 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-21 20:52:44,663 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:52:44,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:44,990 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:45,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:45,068 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:45,618 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:45,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:46,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:47,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:48,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:49,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:50,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:50,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:51,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:52,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:53,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:54,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:55,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:56,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:56,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:57,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:58,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:59,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:59,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:00,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:00,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:01,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:02,139 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:02,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:03,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:04,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:05,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:06,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:07,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:07,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:08,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:09,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:10,955 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:53:10,955 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4 with policy now
+2025-01-21 20:53:10,958 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:53:10,959 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:53:10,959 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:53:10,960 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:53:10,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:11,230 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/kybeb8k0-media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:53:11,626 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:11,626 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:53:11,627 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:53:11,627 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:53:11,923 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/nru49k6z-media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:53:12,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:13,002 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:13,627 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:13,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:14,012 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:14,664 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:53:14,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:14,990 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:15,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:15,628 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:16,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:17,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:18,002 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:19,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:19,365 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:19,629 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:20,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:21,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:22,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:23,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:24,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:24,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:25,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:26,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:26,631 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:53:26,940 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/a7ydzqmp-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:53:27,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:27,461 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:53:27,461 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:53:27,461 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:53:27,632 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:27,632 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:53:28,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:29,006 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:29,632 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:29,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:29,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:30,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:30,137 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:31,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:31,633 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:32,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:33,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:34,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:35,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:35,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:36,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:37,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:38,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:39,006 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:40,006 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:40,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:41,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:42,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:43,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:44,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:44,664 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:53:44,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:44,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:45,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:46,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:46,101 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:47,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:48,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:49,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:50,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:51,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:51,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:52,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:53,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:54,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:55,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:56,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:56,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:57,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:58,011 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:58,612 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:53:58,612 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_660acc7709e365474234.mp4 with policy now
+2025-01-21 20:53:58,615 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:53:58,615 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:53:58,616 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:53:58,616 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:53:58,642 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:53:58,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:53:58,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:53:58,923 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/3bc84cal-media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:53:59,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:59,215 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/nannekno-media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:53:59,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:59,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:59,989 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:00,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:01,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:01,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:02,016 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:02,093 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:03,017 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:03,644 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:04,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:05,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:06,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:07,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:07,645 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:08,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:08,069 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:08,222 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:54:08,222 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:54:08,222 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:54:08,646 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:54:08,646 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:54:09,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:09,646 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:10,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:11,025 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:11,647 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:12,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:13,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:13,166 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:13,648 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:14,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:14,665 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:54:14,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:14,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:15,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:16,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:17,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:18,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:19,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:19,113 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:20,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:21,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:22,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:23,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:24,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:24,957 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:25,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:25,652 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:26,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:27,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:27,652 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:28,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:29,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:29,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:29,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:30,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:30,058 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:31,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:32,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:33,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:34,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:35,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:35,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:36,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:37,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:38,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:39,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:40,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:40,333 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:54:40,333 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4 with policy now
+2025-01-21 20:54:40,334 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:54:40,337 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:54:40,337 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:54:40,338 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:54:40,643 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/ud7n533s-media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:54:40,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:54:40,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:54:40,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:54:40,961 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/1r26uk8z-media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:54:41,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:41,351 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:41,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:42,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:43,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:44,031 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:44,665 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:54:44,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:44,992 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:45,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:45,658 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:46,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:46,737 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:47,031 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:47,659 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:48,032 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:49,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:49,660 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:50,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:51,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:51,275 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:54:51,275 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:54:51,275 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:54:51,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:54:51,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:51,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:54:51,809 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:51,945 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/lum9ldkp-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:54:52,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:53,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:53,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:54,041 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:55,042 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:55,662 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:56,044 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:57,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:57,454 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:57,663 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:58,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:59,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:59,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:59,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:55:00,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:01,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:02,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:03,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:03,147 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:04,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:05,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:06,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:07,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:08,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:08,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:09,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:10,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:11,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:12,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:13,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:13,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:14,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:14,666 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:55:14,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:55:14,989 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:55:15,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:16,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:17,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:18,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:19,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:19,088 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:20,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:21,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:22,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:23,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:24,043 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:24,530 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:24,669 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:55:24,669 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4 with policy now
+2025-01-21 20:55:24,672 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:24,672 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:55:24,674 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:55:24,674 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:24,674 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:24,674 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:24,940 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/r3pa4z1v-media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:25,040 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:25,231 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/hnopdp1d-media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:25,673 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:25,673 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:26,046 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:27,047 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:27,673 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:28,042 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:29,058 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:29,674 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:29,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:55:29,996 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:55:30,058 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:30,062 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:31,049 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:32,052 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:33,056 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:33,675 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:34,056 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:35,069 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:35,674 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:36,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:55:36,007 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:55:36,069 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:36,223 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:55:36,528 INFO    wandb-upload_1:2641882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0isp20kg
+2025-01-21 20:55:36,596 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:55:36,604 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:55:36,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:55:36,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:36,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:36,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media
+2025-01-21 20:55:36,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval
+2025-01-21 20:55:36,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table
+2025-01-21 20:55:36,934 INFO    SenderThread:2641882 [sender.py:send_request_log_artifact():1525] logged artifact run-66rp0oj9-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzMTg3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4ODgzMw==', 'latestArtifact': None}}
+2025-01-21 20:55:36,934 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:55:36,936 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-21 20:55:36,936 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:36,936 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,937 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,937 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:36,937 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,941 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,941 DEBUG   SenderThread:2641882 [sender.py:send():379] send: telemetry
+2025-01-21 20:55:36,941 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,941 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,941 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,942 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,942 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,942 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,942 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,943 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,943 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,943 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,943 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,944 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,944 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,944 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,945 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,946 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,947 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,947 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,948 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,949 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,949 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,950 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,950 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:55:37,061 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:37,123 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/zzxybq3q-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:37,179 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/lj6ssb0a-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:37,340 INFO    wandb-upload_0:2641882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbxh5n5hz
+2025-01-21 20:55:37,394 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/nborh3eh-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:37,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:37,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:37,779 INFO    SenderThread:2641882 [sender.py:send_request_log_artifact():1525] logged artifact run-66rp0oj9-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzMTg5Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4ODgzNg==', 'latestArtifact': None}}
+2025-01-21 20:55:37,779 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:55:37,779 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json with policy now
+2025-01-21 20:55:37,779 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:37,779 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:37,780 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:38,051 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/2a8ebkyz-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:38,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:38,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:39,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:39,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:40,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:41,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:41,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:42,061 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:42,361 DEBUG   SenderThread:2641882 [sender.py:send():379] send: exit
+2025-01-21 20:55:42,361 INFO    SenderThread:2641882 [sender.py:send_exit():586] handling exit code: 0
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:send_exit():588] handling runtime: 237
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:send_exit():594] send defer
+2025-01-21 20:55:42,362 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,362 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-21 20:55:42,362 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-21 20:55:42,363 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 1
+2025-01-21 20:55:42,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-21 20:55:42,363 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,363 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-21 20:55:42,363 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 2
+2025-01-21 20:55:42,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-21 20:55:42,364 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined disk monitor
+2025-01-21 20:55:42,364 DEBUG   SystemMonitor:2641882 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-21 20:55:42,364 DEBUG   SystemMonitor:2641882 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-21 20:55:42,417 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-21 20:55:42,418 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined memory monitor
+2025-01-21 20:55:42,418 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined network monitor
+2025-01-21 20:55:42,418 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,418 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-21 20:55:42,418 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 3
+2025-01-21 20:55:42,418 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:55:42,419 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,419 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-21 20:55:42,419 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,419 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-21 20:55:42,419 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 4
+2025-01-21 20:55:42,419 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,419 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-21 20:55:42,680 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:43,680 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:49,336 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:49,336 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:49,336 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,338 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-21 20:55:49,338 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 5
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:49,339 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:49,339 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:49,339 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-21 20:55:49,340 DEBUG   SenderThread:2641882 [sender.py:send():379] send: summary
+2025-01-21 20:55:49,340 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:49,340 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,341 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-21 20:55:49,341 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 6
+2025-01-21 20:55:49,341 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:49,341 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-21 20:55:49,341 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,341 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-21 20:55:49,344 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:49,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:49,630 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 7
+2025-01-21 20:55:49,630 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:49,630 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:49,630 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-21 20:55:49,630 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,630 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-21 20:55:49,682 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
+2025-01-21 20:55:49,683 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:50,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:53,081 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 8
+2025-01-21 20:55:53,081 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:53,081 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:53,081 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-21 20:55:53,081 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:53,081 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-21 20:55:53,081 INFO    SenderThread:2641882 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-21 20:55:53,084 INFO    SenderThread:2641882 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-21 20:55:53,086 INFO    SenderThread:2641882 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-21 20:55:53,100 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 9
+2025-01-21 20:55:53,101 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:53,101 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-21 20:55:53,101 DEBUG   SenderThread:2641882 [sender.py:send():379] send: artifact
+2025-01-21 20:55:53,314 INFO    SenderThread:2641882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}}
+2025-01-21 20:55:53,315 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:53,315 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-21 20:55:53,315 INFO    SenderThread:2641882 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-21 20:55:53,364 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml config.yaml
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json wandb-summary.json
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt requirements.txt
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json wandb-metadata.json
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0 events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:55:53,686 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log output.log
+2025-01-21 20:55:53,686 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch diff.patch
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4 media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4 media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4 media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:55:53,692 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4 media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4 media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 10
+2025-01-21 20:55:53,693 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:53,693 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:53,694 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-21 20:55:53,694 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:53,694 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-21 20:55:53,694 INFO    SenderThread:2641882 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:55:53,886 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt
+2025-01-21 20:55:53,888 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
+2025-01-21 20:55:53,940 INFO    wandb-upload_4:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:53,944 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:54,000 INFO    wandb-upload_3:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/d75i7fs9-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:55:54,200 INFO    Thread-11 (_thread_body):2641882 [sender.py:transition_state():614] send defer: 11
+2025-01-21 20:55:54,200 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,200 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-21 20:55:54,201 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,201 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-21 20:55:54,201 INFO    SenderThread:2641882 [file_pusher.py:join():175] waiting for file pusher
+2025-01-21 20:55:54,202 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 12
+2025-01-21 20:55:54,202 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,202 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-21 20:55:54,202 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,202 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-21 20:55:54,202 INFO    SenderThread:2641882 [file_stream.py:finish():601] file stream finish called
+2025-01-21 20:55:54,365 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:54,391 INFO    SenderThread:2641882 [file_stream.py:finish():605] file stream finish is done
+2025-01-21 20:55:54,391 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 13
+2025-01-21 20:55:54,391 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:54,391 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,391 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-21 20:55:54,391 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,391 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-21 20:55:54,392 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 14
+2025-01-21 20:55:54,392 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,392 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-21 20:55:54,392 DEBUG   SenderThread:2641882 [sender.py:send():379] send: final
+2025-01-21 20:55:54,392 DEBUG   SenderThread:2641882 [sender.py:send():379] send: footer
+2025-01-21 20:55:54,392 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,392 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-21 20:55:54,393 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:54,393 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:54,393 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: server_info
+2025-01-21 20:55:54,393 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: server_info
+2025-01-21 20:55:54,394 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:54,395 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-21 20:55:54,395 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-21 20:55:54,396 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:54,458 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:54,458 INFO    MainThread:2641882 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-21 20:55:54,459 INFO    MainThread:2641882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-21 20:55:54,459 INFO    MainThread:2641882 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-21 20:55:54,459 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-21 20:55:54,459 INFO    HandlerThread:2641882 [handler.py:finish():882] shutting down handler
+2025-01-21 20:55:55,395 INFO    WriterThread:2641882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb
+2025-01-21 20:55:55,458 INFO    SenderThread:2641882 [sender.py:finish():1615] shutting down sender
+2025-01-21 20:55:55,458 INFO    SenderThread:2641882 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:55:55,458 INFO    SenderThread:2641882 [file_pusher.py:join():175] waiting for file pusher
diff --git a/66rp0oj9/wandb/debug.log b/66rp0oj9/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..fa6c123910f2e045a2dd3e08c0ef2eba2a1330b0
--- /dev/null
+++ b/66rp0oj9/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Configure stats pid to 2641715
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/settings
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug.log
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug-internal.log
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_init.py:init():569] calling init triggers
+2025-01-21 20:51:44,287 INFO    MainThread:2641715 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9'}
+2025-01-21 20:51:44,287 INFO    MainThread:2641715 [wandb_init.py:init():619] starting backend
+2025-01-21 20:51:44,287 INFO    MainThread:2641715 [wandb_init.py:init():623] setting up manager
+2025-01-21 20:51:44,291 INFO    MainThread:2641715 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-21 20:51:44,292 INFO    MainThread:2641715 [wandb_init.py:init():631] backend started and connected
+2025-01-21 20:51:44,303 INFO    MainThread:2641715 [wandb_init.py:init():720] updated telemetry
+2025-01-21 20:51:44,312 INFO    MainThread:2641715 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-21 20:51:44,610 INFO    MainThread:2641715 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-21 20:51:44,649 INFO    MainThread:2641715 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-21 20:51:44,649 INFO    MainThread:2641715 [wandb_init.py:init():804] starting run threads in backend
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-21 20:51:44,989 INFO    MainThread:2641715 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-21 20:51:44,989 INFO    MainThread:2641715 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/tb, True
+2025-01-21 20:51:48,616 INFO    MainThread:2641715 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-21 20:55:55,460 WARNING MsgRouterThr:2641715 [router.py:message_loop():77] message_loop has been closed
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py
@@ -0,0 +1,606 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config), 
+				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				# self.cfg.agent.config.policy_cfg.input_shapes,
+				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0c32086077d7e43d84403a6b3224d052a6f42587
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
@@ -0,0 +1,927 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '3465_0'
+wandb_notes:
+  desc: null
+  value: 3465_0_restarted_1
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: false
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+restore_missing_checkpoint:
+  desc: null
+  value: null
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        crop_distractors_tlhw:
+        - 32
+        - 68
+        - 174
+        - 174
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: false
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+        crop_distractors_tlhw: null
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          crop_distractors_tlhw:
+          - 32
+          - 68
+          - 174
+          - 174
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+test:
+  desc: null
+  value:
+    username: leonmkim
+    hostname: grasp-login1
+    name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+    root_dir: ~/fish_leon
+    expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+    expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+    semantic_demo_grouping_name: semantic_demo_grouping.yaml
+    semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+    include_groups_list: all
+    batch_size: 128
+    store_dataset_in_memory: false
+    use_tb: true
+    local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup
+    local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1
+    resume_wandb_run: false
+feature_type:
+  desc: null
+  value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 7500
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1737510704
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..8da10724284d70d8fca0dedf572670b3116372fc
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch
@@ -0,0 +1,28 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb5fa99..e77ad21 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17
+ 
+ # Number of evaluation trajectories
+ # num_eval: 10
+-num_eval: 20
++num_eval: 5
+ 
+ debug_timestamps: False
+ open_loop: False
+@@ -139,12 +139,12 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 06/20 demos per book
+ # crp_D+mask+act history
+-# wandb_run_id: '3465_0'
++wandb_run_id: '3465_0'
+ # wandb_run_id: '3465_1' # seed 1
+ # wandb_run_id: '3465_2' # seed 2
+ 
+ # crp_D+mask+contact+act history
+-wandb_run_id: '4295_0'
++# wandb_run_id: '4295_0'
+ # wandb_run_id: '4295_1' # seed 1
+ # wandb_run_id: '4295_2' # seed 2
+ 
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..0688daf9c9e63e02548e7f2b22545029ebfbf063
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[48, 50, 0.96, 0.8497122823492624, 0.9960396214327554, 0.8803341164424305, 0.9949651741364105, 0.8956393226189883, 0.9885692316061636, 0.9012663065584643, 0.9885647114320486]]}
\ No newline at end of file
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4 b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bbf13b25e8ba984cd867973e1150a8575cb1239c
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10f2d5a1476b38f259e0bd9786e10425512ac938efc84e83d6a0cf7e00e4d860
+size 1173896
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4 b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4973064d55e977d23ed7fed749b82d358bf795e6
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d14dcbccf52d3b425c83b0b193b4d4335542e2a2655a13a02e11d4b72394c121
+size 1173762
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4 b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4f05d4c40bd71d49a2f1324c0431a3a7196fc3c3
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:660acc7709e365474234422a1637eca0b03396ba0dd76d4a713e46bd3597a440
+size 1188354
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4 b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ebf893f14c735464376fdf3401be6e9154b39e4f
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf508a8824a44b8c6adbba807899817a3119200de11972a46b32b83dc94d868d
+size 1182542
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4 b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..502dd62bb842ec7b17464d890b95b5d7307ce1f7
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75f1e1972486326d53ed64a7ae2ba2f5db26aa29381c7d76bc21f2944f5f06be
+size 1237160
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..bee578c2c18c2849f9b3445d06b5ef9d2ebb7cae
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
@@ -0,0 +1,281 @@
+
+loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1737510708.616719]: resetting environment
+[INFO] [1737510708.620018]: cleared current plan
+[INFO] [1737510708.620863]: moving to home
+[INFO] [1737510709.623182]: reached home
+[INFO] [1737510709.624408]: reset action history
+[INFO] [1737510712.338085]: environment reset
+Starting episode 0
+[INFO] [1737510712.341696]: resetting environment
+[INFO] [1737510712.344517]: cleared current plan
+[INFO] [1737510712.345440]: moving to home
+[INFO] [1737510713.347625]: reached home
+[INFO] [1737510713.349032]: reset action history
+[INFO] [1737510716.057109]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1737510717.640123]: resetting environment
+[INFO] [1737510717.643337]: cleared current plan
+[INFO] [1737510717.644178]: moving to home
+[INFO] [1737510718.646808]: reached home
+[INFO] [1737510718.648151]: reset action history
+[INFO] [1737510721.353995]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/episode_rosbags/episode_0_2025-01-21-20-52-01.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2641715.log
+started roslaunch server http://158.130.50.37:46557/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2642003]
+process[print_text-2]: started with pid [2642004]
+process[rosbag_record-3]: started with pid [2642005]
+started bagging!
+For topic gripper_width: timestamp difference is 111524948 for nearest: 1737510724979205021 - target: 1737510724867680073 at idx 51
+[INFO] [1737510748.355613]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1737510749.105639]: Stored episode 1.
+[INFO] [1737510749.106025]: Saving video...
+[INFO] [1737510749.309507]: Video saved!
+Starting episode 1
+[INFO] [1737510749.322048]: resetting environment
+[INFO] [1737510749.337732]: cleared current plan
+[INFO] [1737510749.338477]: moving to home
+[INFO] [1737510752.943972]: reached home
+[INFO] [1737510752.944472]: reset action history
+[INFO] [1737510755.650489]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510758.908625]: resetting environment
+[INFO] [1737510758.910895]: cleared current plan
+[INFO] [1737510758.911055]: moving to home
+[INFO] [1737510759.912572]: reached home
+[INFO] [1737510759.913194]: reset action history
+[INFO] [1737510762.621839]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/episode_rosbags/episode_1_2025-01-21-20-52-42.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2641715.log
+started roslaunch server http://158.130.50.37:44233/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2642127]
+process[print_text-5]: started with pid [2642128]
+process[rosbag_record-6]: started with pid [2642152]
+started bagging!
+[INFO] [1737510789.624360]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1737510790.352463]: Stored episode 2.
+[INFO] [1737510790.353299]: Saving video...
+[WARN] [1737510790.927974]: Plan exhausted
+[WARN] [1737510790.967653]: Plan exhausted
+[INFO] [1737510790.922464]: Video saved!
+Starting episode 2
+[INFO] [1737510790.970392]: resetting environment
+[INFO] [1737510790.986933]: cleared current plan
+[INFO] [1737510790.987406]: moving to home
+[INFO] [1737510794.593422]: reached home
+[INFO] [1737510794.596406]: reset action history
+[INFO] [1737510797.322702]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510806.402728]: resetting environment
+[INFO] [1737510806.409188]: cleared current plan
+[INFO] [1737510806.409384]: moving to home
+[INFO] [1737510807.411285]: reached home
+[INFO] [1737510807.414294]: reset action history
+[INFO] [1737510810.122330]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/episode_rosbags/episode_2_2025-01-21-20-53-30.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2641715.log
+started roslaunch server http://158.130.50.37:40069/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2642297]
+process[print_text-8]: started with pid [2642298]
+process[rosbag_record-9]: started with pid [2642299]
+started bagging!
+[WARN] [1737510838.408489]: Plan exhausted
+[WARN] [1737510838.458102]: Plan exhausted
+[INFO] [1737510837.251764]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1737510838.024114]: Stored episode 3.
+[INFO] [1737510838.027730]: Saving video...
+[INFO] [1737510838.585111]: Video saved!
+Starting episode 3
+[INFO] [1737510838.616828]: resetting environment
+[INFO] [1737510838.636763]: cleared current plan
+[INFO] [1737510838.637095]: moving to home
+[WARN] [1737510838.493636]: Plan exhausted
+[WARN] [1737510838.529829]: Plan exhausted
+[WARN] [1737510838.567502]: Plan exhausted
+[WARN] [1737510838.610769]: Plan exhausted
+[INFO] [1737510842.239466]: reached home
+[INFO] [1737510842.245255]: reset action history
+[INFO] [1737510845.023319]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510848.091761]: resetting environment
+[INFO] [1737510848.100659]: cleared current plan
+[INFO] [1737510848.101047]: moving to home
+[INFO] [1737510849.102907]: reached home
+[INFO] [1737510849.105001]: reset action history
+[INFO] [1737510851.885465]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/episode_rosbags/episode_3_2025-01-21-20-54-11.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2641715.log
+started roslaunch server http://158.130.50.37:38687/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2642419]
+process[print_text-11]: started with pid [2642420]
+process[rosbag_record-12]: started with pid [2642421]
+started bagging!
+For topic gripper_width: timestamp difference is 112041806 for nearest: 1737510862912525271 - target: 1737510862800483465 at idx 150
+For topic gripper_width: timestamp difference is -119445629 for nearest: 1737510865112535410 - target: 1737510865231981039 at idx 177
+[WARN] [1737510880.250884]: Plan exhausted
+[WARN] [1737510880.288409]: Plan exhausted
+[WARN] [1737510880.329223]: Plan exhausted
+[INFO] [1737510878.878492]: Storing episode...
+[rosbag_record-12] killing on exit
+[pub_text-10] killing on exit
+[print_text-11] killing on exit
+[INFO] [1737510879.637875]: Stored episode 4.
+[INFO] [1737510879.639310]: Saving video...
+[INFO] [1737510880.295757]: Video saved!
+Starting episode 4
+[INFO] [1737510880.339170]: resetting environment
+[INFO] [1737510880.350207]: cleared current plan
+[INFO] [1737510880.350451]: moving to home
+[INFO] [1737510883.953290]: reached home
+[INFO] [1737510883.959036]: reset action history
+[INFO] [1737510886.737274]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510890.787867]: resetting environment
+[INFO] [1737510890.799198]: cleared current plan
+[INFO] [1737510890.802418]: moving to home
+[INFO] [1737510891.807481]: reached home
+[INFO] [1737510891.811599]: reset action history
+[INFO] [1737510894.583997]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/episode_rosbags/episode_4_2025-01-21-20-54-54.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2641715.log
+started roslaunch server http://158.130.50.37:41157/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2642576]
+process[print_text-14]: started with pid [2642577]
+process[rosbag_record-15]: started with pid [2642578]
+started bagging!
+[WARN] [1737510924.527793]: Plan exhausted
+[INFO] [1737510923.161141]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1737510923.931071]: Stored episode 5.
+[INFO] [1737510923.932574]: Saving video...
+[INFO] [1737510924.632732]: Video saved!
+[INFO] [1737510924.674843]: resetting environment
+[INFO] [1737510924.696053]: cleared current plan
+[INFO] [1737510924.696518]: moving to home
+[WARN] [1737510924.569199]: Plan exhausted
+[WARN] [1737510924.608004]: Plan exhausted
+[WARN] [1737510924.647908]: Plan exhausted
+[WARN] [1737510924.687842]: Plan exhausted
+[INFO] [1737510928.304834]: reached home
+[INFO] [1737510928.314279]: reset action history
+[INFO] [1737510931.125091]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..e716cc788cacd4d2b9645c578cf7531f25ace928
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-22T01:51:44.677333",
+    "startedAt": "2025-01-22T01:51:44.284707",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 4122.9805,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4497.584,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 322.3823585510254
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..e562ce12887b0443543b586920649335a3159a4d
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "10f2d5a1476b38f259e0bd9786e10425512ac938efc84e83d6a0cf7e00e4d860", "size": 1173896, "path": "media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4"}, "global_step": 663, "_timestamp": 1737510936.5154562, "_runtime": 232.22303318977356, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d14dcbccf52d3b425c83b0b193b4d4335542e2a2655a13a02e11d4b72394c121", "size": 1173762, "path": "media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "660acc7709e365474234422a1637eca0b03396ba0dd76d4a713e46bd3597a440", "size": 1188354, "path": "media/videos/eval/2_eval_2_660acc7709e365474234.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "bf508a8824a44b8c6adbba807899817a3119200de11972a46b32b83dc94d868d", "size": 1182542, "path": "media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "75f1e1972486326d53ed64a7ae2ba2f5db26aa29381c7d76bc21f2944f5f06be", "size": 1237160, "path": "media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://2ciqcchxk6z6sroqkhfhokfwjcudtogl7b2q5mp1o5wql2o82gmaba7prvf5j9vijcdbcnmx2ikzenyhh9ywxb0lck83hjt0v794rq5g4j42xo9xzod4ecvk25y017ss/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3dcdedpsszxs3n8g78oqkvt3ckp80211176za0qgd5eu349owh8p22wtqmwpjdhef8eimngwmatoxzeb0ha31uhgvbs5q077tp4lq5fhrbf832h8ynek6ikfpa1z5tek:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 48, "total_num_episodes": 50, "total_success_rate": 0.96, "total_success_rate_lb_0.01": 0.8497122823492624, "total_success_rate_ub_0.01": 0.9960396214327554, "total_success_rate_lb_0.025": 0.8803341164424305, "total_success_rate_ub_0.025": 0.9949651741364105, "total_success_rate_lb_0.05": 0.8956393226189883, "total_success_rate_ub_0.05": 0.9885692316061636, "total_success_rate_lb_0.1": 0.9012663065584643, "total_success_rate_ub_0.1": 0.9885647114320486, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2ba98cbc4fea23f33f1441c6998e15f8238ad0040a1049ed00e1b5d253c65630", "size": 506, "artifact_path": "wandb-client-artifact://6p9rrv8raiqh7kszonzjvcrz2m256wq9f9zlmv9ff4c1xn56d9yti62ac78ula3j6tmu0hie2s0iyw8gd2u6h2jrjsaajzi44xwjscjffx8uh6h3zd224upfkh1tj79v/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://6ibx8ifrkyzlqb5a72a0s7iuvvukewellfshn1gttql1scl12dbp4qejdj2fyodj4k3agdxku94ujlyc26wgw1v0hkujwnwtexycuv8dldl4wzbmfnq0v67jhevvzuhk:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 237}, "eval/total_success_rate": 0.9599999785423279, "eval/total_success_rate_lb_0.01": 0.8497122526168823, "eval/total_success_rate_ub_0.01": 0.996039628982544, "eval/total_success_rate_lb_0.025": 0.8803341388702393, "eval/total_success_rate_ub_0.025": 0.9949651956558228, "eval/total_success_rate_lb_0.05": 0.8956393003463745, "eval/total_success_rate_ub_0.05": 0.9885692596435547, "eval/total_success_rate_lb_0.1": 0.9012662768363953, "eval/total_success_rate_ub_0.1": 0.9885647296905518}
\ No newline at end of file
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug-internal.log b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c8944505778f13666ee039b8730e7431cda6f7e7
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug-internal.log
@@ -0,0 +1,729 @@
+2025-01-21 20:51:44,292 INFO    StreamThr :2641882 [internal.py:wandb_internal():85] W&B internal server running at pid: 2641882, started at: 2025-01-21 20:51:44.292377
+2025-01-21 20:51:44,293 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status
+2025-01-21 20:51:44,304 INFO    WriterThread:2641882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb
+2025-01-21 20:51:44,305 DEBUG   SenderThread:2641882 [sender.py:send():379] send: header
+2025-01-21 20:51:44,315 DEBUG   SenderThread:2641882 [sender.py:send():379] send: run
+2025-01-21 20:51:44,594 INFO    SenderThread:2641882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files
+2025-01-21 20:51:44,594 INFO    SenderThread:2641882 [sender.py:_start_run_threads():1188] run started: 66rp0oj9 with start time 1737510704.292423
+2025-01-21 20:51:44,611 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: check_version
+2025-01-21 20:51:44,611 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: check_version
+2025-01-21 20:51:44,654 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: run_start
+2025-01-21 20:51:44,659 DEBUG   HandlerThread:2641882 [system_info.py:__init__():26] System info init
+2025-01-21 20:51:44,659 DEBUG   HandlerThread:2641882 [system_info.py:__init__():41] System info init done
+2025-01-21 20:51:44,659 INFO    HandlerThread:2641882 [system_monitor.py:start():194] Starting system monitor
+2025-01-21 20:51:44,659 INFO    SystemMonitor:2641882 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-21 20:51:44,659 INFO    HandlerThread:2641882 [system_monitor.py:probe():214] Collecting system info
+2025-01-21 20:51:44,660 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started cpu monitoring
+2025-01-21 20:51:44,661 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started disk monitoring
+2025-01-21 20:51:44,661 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started gpu monitoring
+2025-01-21 20:51:44,662 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started memory monitoring
+2025-01-21 20:51:44,662 INFO    SystemMonitor:2641882 [interfaces.py:start():188] Started network monitoring
+2025-01-21 20:51:44,677 DEBUG   HandlerThread:2641882 [system_info.py:probe():152] Probing system
+2025-01-21 20:51:44,679 DEBUG   HandlerThread:2641882 [system_info.py:_probe_git():137] Probing git
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_info.py:_probe_git():145] Probing git done
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_info.py:probe():200] Probing system done
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:51:44.677333', 'startedAt': '2025-01-22T01:51:44.284707', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4122.9805, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.584, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 322.3823585510254}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-21 20:51:44,684 INFO    HandlerThread:2641882 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-21 20:51:44,684 INFO    HandlerThread:2641882 [system_monitor.py:probe():227] Publishing system info
+2025-01-21 20:51:44,684 DEBUG   HandlerThread:2641882 [system_info.py:_save_code():44] Saving code
+2025-01-21 20:51:44,686 DEBUG   HandlerThread:2641882 [system_info.py:_save_code():67] Saving code done
+2025-01-21 20:51:44,686 DEBUG   HandlerThread:2641882 [system_info.py:_save_patches():84] Saving git patches
+2025-01-21 20:51:44,747 DEBUG   HandlerThread:2641882 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-21 20:51:44,747 INFO    HandlerThread:2641882 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-21 20:51:44,750 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:51:44,750 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-21 20:51:44,751 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-21 20:51:44,751 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-21 20:51:44,986 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-21 20:51:44,986 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: python_packages
+2025-01-21 20:51:44,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:51:44,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:51:44,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:44,990 INFO    HandlerThread:2641882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/tb"
+  save: true
+}
+control {
+  relay_id: "12c704b22f184c97abe0deaf0658d3b2"
+}
+_info {
+  stream_id: "66rp0oj9"
+}
+
+2025-01-21 20:51:44,990 DEBUG   HandlerThread:2641882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-21 20:51:45,020 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/j27huxi2-wandb-metadata.json
+2025-01-21 20:51:45,052 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/6mgydkap-diff.patch
+2025-01-21 20:51:45,062 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/ksqjbenx-code/FISH/eval_robot.py
+2025-01-21 20:51:45,069 DEBUG   SenderThread:2641882 [sender.py:send():379] send: telemetry
+2025-01-21 20:51:45,069 DEBUG   SenderThread:2641882 [sender.py:send():379] send: tbrecord
+2025-01-21 20:51:45,070 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:51:45,070 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0 with policy live
+2025-01-21 20:51:45,295 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/6wiua7gr-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:51:45,601 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch
+2025-01-21 20:51:45,601 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json
+2025-01-21 20:51:45,601 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code
+2025-01-21 20:51:45,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH
+2025-01-21 20:51:45,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:46,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:47,600 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:47,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:48,616 DEBUG   SenderThread:2641882 [sender.py:send():379] send: config
+2025-01-21 20:51:48,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:49,623 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:51:49,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:50,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:51,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:51,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:52,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:53,602 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:53,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:54,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:51:54,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:55,603 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:55,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:56,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:57,603 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:57,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:58,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:51:59,604 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:51:59,986 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:51:59,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:00,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:00,080 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:00,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:01,605 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:01,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:02,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:03,605 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:03,990 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:05,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:05,990 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:06,034 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:06,990 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:07,607 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:07,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:08,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:09,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:10,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:11,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:11,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:12,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:13,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:14,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:14,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:15,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:15,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:16,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:17,128 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:17,609 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
+2025-01-21 20:52:17,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:18,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:19,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:20,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:21,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:22,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:22,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:23,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:24,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:25,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:26,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:27,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:27,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:28,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:29,320 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:52:29,320 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4 with policy now
+2025-01-21 20:52:29,320 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:52:29,321 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:52:29,322 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:52:29,322 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:52:29,587 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/qp87anz1-media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:52:29,612 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos
+2025-01-21 20:52:29,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:52:29,906 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/3mze40wd-media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:52:29,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:29,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:29,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:30,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:31,613 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:31,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:32,994 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:33,985 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:33,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:34,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:35,614 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:35,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:36,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:37,615 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:37,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:38,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:39,616 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:52:39,913 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:39,995 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:40,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:41,616 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:41,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:42,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:43,617 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:43,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:44,663 DEBUG   SystemMonitor:2641882 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-21 20:52:44,663 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:52:44,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:44,990 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:52:45,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:45,068 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:45,618 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:52:45,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:46,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:47,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:48,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:49,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:50,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:50,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:51,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:52,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:53,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:54,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:55,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:56,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:52:56,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:57,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:58,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:52:59,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:52:59,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:00,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:00,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:01,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:02,139 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:02,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:03,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:04,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:05,998 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:06,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:07,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:07,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:08,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:09,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:10,955 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:53:10,955 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4 with policy now
+2025-01-21 20:53:10,958 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:53:10,959 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:53:10,959 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:53:10,960 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:53:10,999 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:11,230 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/kybeb8k0-media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:53:11,626 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:11,626 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:53:11,627 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:53:11,627 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:53:11,923 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/nru49k6z-media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:53:12,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:13,002 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:13,627 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:13,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:14,012 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:14,664 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:53:14,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:14,990 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:15,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:15,628 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:16,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:17,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:18,002 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:19,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:19,365 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:19,629 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:20,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:21,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:22,003 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:23,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:24,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:24,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:25,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:26,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:26,631 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:53:26,940 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/a7ydzqmp-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:53:27,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:27,461 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:53:27,461 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:53:27,461 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:53:27,632 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:27,632 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:53:28,004 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:29,006 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:29,632 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:29,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:29,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:30,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:30,137 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:31,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:31,633 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:32,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:33,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:34,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:35,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:35,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:36,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:37,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:38,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:39,006 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:40,006 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:40,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:41,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:42,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:43,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:44,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:44,664 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:53:44,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:44,987 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:53:45,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:46,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:46,101 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:47,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:48,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:49,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:50,008 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:51,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:51,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:52,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:53,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:54,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:55,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:56,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:56,991 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:53:57,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:58,011 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:58,612 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:53:58,612 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_660acc7709e365474234.mp4 with policy now
+2025-01-21 20:53:58,615 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:53:58,615 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:53:58,616 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:53:58,616 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:53:58,642 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:53:58,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:53:58,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:53:58,923 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/3bc84cal-media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:53:59,009 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:53:59,215 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/nannekno-media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:53:59,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:53:59,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:53:59,989 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:00,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:01,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:01,643 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:02,016 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:02,093 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:03,017 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:03,644 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:04,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:05,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:06,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:07,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:07,645 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:08,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:08,069 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:08,222 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:54:08,222 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:54:08,222 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:54:08,646 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:54:08,646 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:54:09,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:09,646 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:10,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:11,025 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:11,647 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:12,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:13,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:13,166 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:13,648 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:14,018 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:14,665 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:54:14,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:14,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:15,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:16,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:17,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:18,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:19,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:19,113 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:20,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:21,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:22,019 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:23,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:24,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:24,957 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:25,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:25,652 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:26,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:27,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:27,652 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:28,020 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:29,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:29,987 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:29,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:30,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:30,058 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:31,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:32,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:33,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:34,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:35,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:35,993 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:36,021 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:37,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:38,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:39,022 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:40,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:40,333 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:54:40,333 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4 with policy now
+2025-01-21 20:54:40,334 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:54:40,337 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:54:40,337 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:54:40,338 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:54:40,643 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/ud7n533s-media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:54:40,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:54:40,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:54:40,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:54:40,961 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/1r26uk8z-media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:54:41,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:41,351 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:41,657 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:42,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:43,029 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:44,031 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:44,665 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:54:44,992 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:44,992 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:54:45,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:45,658 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:46,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:46,737 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:47,031 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:47,659 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:48,032 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:49,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:49,660 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:50,033 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:51,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:51,275 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:54:51,275 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:54:51,275 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:54:51,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:54:51,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:51,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:54:51,809 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:51,945 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/lum9ldkp-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:54:52,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:53,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:53,661 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:54,041 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:55,042 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:55,662 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:56,044 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:57,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:57,454 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:54:57,663 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:54:58,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:59,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:54:59,988 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:54:59,988 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:55:00,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:01,036 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:02,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:03,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:03,147 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:04,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:05,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:06,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:07,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:08,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:08,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:09,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:10,037 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:11,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:12,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:13,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:13,997 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:14,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:14,666 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:55:14,989 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:55:14,989 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:55:15,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:16,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:17,038 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:18,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:19,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:19,088 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:20,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:21,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:22,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:23,039 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:24,043 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:24,530 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:24,669 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:55:24,669 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4 with policy now
+2025-01-21 20:55:24,672 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:24,672 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval
+2025-01-21 20:55:24,674 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:55:24,674 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:24,674 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:24,674 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:24,940 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/r3pa4z1v-media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:25,040 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:25,231 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/hnopdp1d-media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:25,673 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:25,673 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:26,046 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:27,047 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:27,673 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:28,042 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:29,058 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:29,674 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:29,996 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:55:29,996 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:55:30,058 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:30,062 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:31,049 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:32,052 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:33,056 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:33,675 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:34,056 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:35,069 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:35,674 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:36,007 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:55:36,007 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:55:36,069 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:36,223 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:55:36,528 INFO    wandb-upload_1:2641882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0isp20kg
+2025-01-21 20:55:36,596 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:55:36,604 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:55:36,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:55:36,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:36,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:36,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media
+2025-01-21 20:55:36,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval
+2025-01-21 20:55:36,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table
+2025-01-21 20:55:36,934 INFO    SenderThread:2641882 [sender.py:send_request_log_artifact():1525] logged artifact run-66rp0oj9-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzMTg3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4ODgzMw==', 'latestArtifact': None}}
+2025-01-21 20:55:36,934 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:55:36,936 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-21 20:55:36,936 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:36,936 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,937 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,937 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:36,937 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,941 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,941 DEBUG   SenderThread:2641882 [sender.py:send():379] send: telemetry
+2025-01-21 20:55:36,941 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,941 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,941 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,942 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,942 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,942 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,942 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,943 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,943 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,943 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,943 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,944 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,944 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,944 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,945 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,946 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,947 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,947 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,948 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,949 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,949 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:36,950 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:36,950 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:55:37,061 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:37,123 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/zzxybq3q-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:37,179 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/lj6ssb0a-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:37,340 INFO    wandb-upload_0:2641882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbxh5n5hz
+2025-01-21 20:55:37,394 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/nborh3eh-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:37,677 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:37,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:37,779 INFO    SenderThread:2641882 [sender.py:send_request_log_artifact():1525] logged artifact run-66rp0oj9-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzMTg5Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4ODgzNg==', 'latestArtifact': None}}
+2025-01-21 20:55:37,779 DEBUG   SenderThread:2641882 [sender.py:send():379] send: files
+2025-01-21 20:55:37,779 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json with policy now
+2025-01-21 20:55:37,779 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:37,779 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:37,780 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:38,051 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/2a8ebkyz-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:38,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:38,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:39,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:39,678 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:40,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:41,005 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:41,060 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:42,061 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:42,361 DEBUG   SenderThread:2641882 [sender.py:send():379] send: exit
+2025-01-21 20:55:42,361 INFO    SenderThread:2641882 [sender.py:send_exit():586] handling exit code: 0
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:send_exit():588] handling runtime: 237
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:send_exit():594] send defer
+2025-01-21 20:55:42,362 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,362 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-21 20:55:42,362 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,362 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-21 20:55:42,363 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 1
+2025-01-21 20:55:42,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-21 20:55:42,363 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,363 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-21 20:55:42,363 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 2
+2025-01-21 20:55:42,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-21 20:55:42,363 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-21 20:55:42,364 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined disk monitor
+2025-01-21 20:55:42,364 DEBUG   SystemMonitor:2641882 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-21 20:55:42,364 DEBUG   SystemMonitor:2641882 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-21 20:55:42,417 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-21 20:55:42,418 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined memory monitor
+2025-01-21 20:55:42,418 INFO    HandlerThread:2641882 [interfaces.py:finish():200] Joined network monitor
+2025-01-21 20:55:42,418 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,418 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-21 20:55:42,418 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 3
+2025-01-21 20:55:42,418 DEBUG   SenderThread:2641882 [sender.py:send():379] send: stats
+2025-01-21 20:55:42,419 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,419 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-21 20:55:42,419 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:42,419 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-21 20:55:42,419 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 4
+2025-01-21 20:55:42,419 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:42,419 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-21 20:55:42,680 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:43,680 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:49,336 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:49,336 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:49,336 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,338 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-21 20:55:49,338 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 5
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send():379] send: history
+2025-01-21 20:55:49,338 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:55:49,339 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:49,339 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:49,339 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-21 20:55:49,340 DEBUG   SenderThread:2641882 [sender.py:send():379] send: summary
+2025-01-21 20:55:49,340 INFO    SenderThread:2641882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:55:49,340 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,341 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-21 20:55:49,341 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 6
+2025-01-21 20:55:49,341 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:49,341 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-21 20:55:49,341 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,341 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-21 20:55:49,344 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:55:49,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:49,630 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 7
+2025-01-21 20:55:49,630 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:49,630 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:49,630 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-21 20:55:49,630 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:49,630 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-21 20:55:49,682 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
+2025-01-21 20:55:49,683 INFO    Thread-12 :2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:50,363 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:53,081 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 8
+2025-01-21 20:55:53,081 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:53,081 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:53,081 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-21 20:55:53,081 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:53,081 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-21 20:55:53,081 INFO    SenderThread:2641882 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-21 20:55:53,084 INFO    SenderThread:2641882 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-21 20:55:53,086 INFO    SenderThread:2641882 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-21 20:55:53,100 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 9
+2025-01-21 20:55:53,101 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:53,101 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-21 20:55:53,101 DEBUG   SenderThread:2641882 [sender.py:send():379] send: artifact
+2025-01-21 20:55:53,314 INFO    SenderThread:2641882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}}
+2025-01-21 20:55:53,315 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:53,315 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-21 20:55:53,315 INFO    SenderThread:2641882 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-21 20:55:53,364 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml config.yaml
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json wandb-summary.json
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt requirements.txt
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-metadata.json wandb-metadata.json
+2025-01-21 20:55:53,684 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0 events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:55:53,686 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log output.log
+2025-01-21 20:55:53,686 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/diff.patch diff.patch
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4 media/videos/eval/1_eval_1_d14dcbccf52d3b425c83.mp4
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4 media/videos/eval/4_eval_4_75f1e1972486326d53ed.mp4
+2025-01-21 20:55:53,690 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4 media/videos/eval/0_eval_0_10f2d5a1476b38f259e0.mp4
+2025-01-21 20:55:53,692 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4 media/videos/eval/3_eval_3_bf508a8824a44b8c6adb.mp4
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/videos/eval/2_eval_2_660acc7709e365474234.mp4 media/videos/eval/2_eval_2_660acc7709e365474234.mp4
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:55:53,693 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 10
+2025-01-21 20:55:53,693 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:53,693 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:53,694 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-21 20:55:53,694 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:53,694 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-21 20:55:53,694 INFO    SenderThread:2641882 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:55:53,886 INFO    wandb-upload_2:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/requirements.txt
+2025-01-21 20:55:53,888 INFO    wandb-upload_1:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/config.yaml
+2025-01-21 20:55:53,940 INFO    wandb-upload_4:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/output.log
+2025-01-21 20:55:53,944 INFO    wandb-upload_0:2641882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/files/wandb-summary.json
+2025-01-21 20:55:54,000 INFO    wandb-upload_3:2641882 [upload_job.py:push():130] Uploaded file /tmp/tmpcjc06da4wandb/d75i7fs9-events.out.tfevents.1737510704.leonmkim-ROG-Strix-G15CS-G15CS.2641715.0
+2025-01-21 20:55:54,200 INFO    Thread-11 (_thread_body):2641882 [sender.py:transition_state():614] send defer: 11
+2025-01-21 20:55:54,200 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,200 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-21 20:55:54,201 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,201 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-21 20:55:54,201 INFO    SenderThread:2641882 [file_pusher.py:join():175] waiting for file pusher
+2025-01-21 20:55:54,202 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 12
+2025-01-21 20:55:54,202 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,202 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-21 20:55:54,202 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,202 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-21 20:55:54,202 INFO    SenderThread:2641882 [file_stream.py:finish():601] file stream finish called
+2025-01-21 20:55:54,365 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:54,391 INFO    SenderThread:2641882 [file_stream.py:finish():605] file stream finish is done
+2025-01-21 20:55:54,391 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 13
+2025-01-21 20:55:54,391 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:54,391 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,391 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-21 20:55:54,391 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,391 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-21 20:55:54,392 INFO    SenderThread:2641882 [sender.py:transition_state():614] send defer: 14
+2025-01-21 20:55:54,392 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:55:54,392 INFO    HandlerThread:2641882 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-21 20:55:54,392 DEBUG   SenderThread:2641882 [sender.py:send():379] send: final
+2025-01-21 20:55:54,392 DEBUG   SenderThread:2641882 [sender.py:send():379] send: footer
+2025-01-21 20:55:54,392 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:55:54,392 INFO    SenderThread:2641882 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-21 20:55:54,393 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:54,393 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:54,393 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: server_info
+2025-01-21 20:55:54,393 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: server_info
+2025-01-21 20:55:54,394 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:55:54,395 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-21 20:55:54,395 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-21 20:55:54,396 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:55:54,458 DEBUG   SenderThread:2641882 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:55:54,458 INFO    MainThread:2641882 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-21 20:55:54,459 INFO    MainThread:2641882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-21 20:55:54,459 INFO    MainThread:2641882 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-21 20:55:54,459 DEBUG   HandlerThread:2641882 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-21 20:55:54,459 INFO    HandlerThread:2641882 [handler.py:finish():882] shutting down handler
+2025-01-21 20:55:55,395 INFO    WriterThread:2641882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb
+2025-01-21 20:55:55,458 INFO    SenderThread:2641882 [sender.py:finish():1615] shutting down sender
+2025-01-21 20:55:55,458 INFO    SenderThread:2641882 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:55:55,458 INFO    SenderThread:2641882 [file_pusher.py:join():175] waiting for file pusher
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug.log b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..fa6c123910f2e045a2dd3e08c0ef2eba2a1330b0
--- /dev/null
+++ b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Configure stats pid to 2641715
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/settings
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug.log
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/logs/debug-internal.log
+2025-01-21 20:51:44,286 INFO    MainThread:2641715 [wandb_init.py:init():569] calling init triggers
+2025-01-21 20:51:44,287 INFO    MainThread:2641715 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9'}
+2025-01-21 20:51:44,287 INFO    MainThread:2641715 [wandb_init.py:init():619] starting backend
+2025-01-21 20:51:44,287 INFO    MainThread:2641715 [wandb_init.py:init():623] setting up manager
+2025-01-21 20:51:44,291 INFO    MainThread:2641715 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-21 20:51:44,292 INFO    MainThread:2641715 [wandb_init.py:init():631] backend started and connected
+2025-01-21 20:51:44,303 INFO    MainThread:2641715 [wandb_init.py:init():720] updated telemetry
+2025-01-21 20:51:44,312 INFO    MainThread:2641715 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-21 20:51:44,610 INFO    MainThread:2641715 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-21 20:51:44,649 INFO    MainThread:2641715 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-21 20:51:44,649 INFO    MainThread:2641715 [wandb_init.py:init():804] starting run threads in backend
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-21 20:51:44,987 INFO    MainThread:2641715 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-21 20:51:44,989 INFO    MainThread:2641715 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-21 20:51:44,989 INFO    MainThread:2641715 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/66rp0oj9/tb, True
+2025-01-21 20:51:48,616 INFO    MainThread:2641715 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-21 20:55:55,460 WARNING MsgRouterThr:2641715 [router.py:message_loop():77] message_loop has been closed
diff --git a/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..13ad6b15be42a580aa5a62b24f56a0605d41ad7b
Binary files /dev/null and b/66rp0oj9/wandb/run-20250121_205144-66rp0oj9/run-66rp0oj9.wandb differ
diff --git a/fbqceefl/.hydra/config.yaml b/fbqceefl/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6f829134d57914397f938447dc055c8c1721578e
--- /dev/null
+++ b/fbqceefl/.hydra/config.yaml
@@ -0,0 +1,352 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '3465_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 7500
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      crop_distractors_tlhw: null
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+      crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw}
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/fbqceefl/.hydra/hydra.yaml b/fbqceefl/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..caa3de752ac91f6e04b3c9b31b39e912f4d4a95e
--- /dev/null
+++ b/fbqceefl/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/fbqceefl
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/fbqceefl/eval_policy.log b/fbqceefl/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..4a86979572898621c90bc06008898bbc16a858b7
--- /dev/null
+++ b/fbqceefl/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-21 21:06:40,671][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-21 21:06:40,674][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-21 21:06:45,032][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/gqds13di/.hydra/config.yaml b/gqds13di/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b1f05e6fe3d0ebd009320b13cbc68f24b3ac1661
--- /dev/null
+++ b/gqds13di/.hydra/config.yaml
@@ -0,0 +1,352 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 20
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '3465_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 7500
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      crop_distractors_tlhw: null
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+      crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw}
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/gqds13di/.hydra/hydra.yaml b/gqds13di/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..606b8bc2822f54ec23e250f19e2d5bbf3fad1395
--- /dev/null
+++ b/gqds13di/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/gqds13di
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/gqds13di/.hydra/overrides.yaml b/gqds13di/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/gqds13di/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/gqds13di/eval_policy.log b/gqds13di/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..f3e6cb2ae9edf966d86b619bf8ccfb66878006ab
--- /dev/null
+++ b/gqds13di/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-18 19:54:23,685][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-18 19:54:23,688][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-18 19:54:24,890][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/mo0b6mh9/episode_rosbags/episode_14_2025-01-18-20-06-53.bag b/mo0b6mh9/episode_rosbags/episode_14_2025-01-18-20-06-53.bag
new file mode 100644
index 0000000000000000000000000000000000000000..8b7e4e0b62122b582e4ff89064e0251fd515e557
--- /dev/null
+++ b/mo0b6mh9/episode_rosbags/episode_14_2025-01-18-20-06-53.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa981cc6eafb8c6e19ac5cd00edf5cff5169939aaa9b489351a240c44363dd02
+size 1411159066
diff --git a/mo0b6mh9/episode_rosbags/episode_19_2025-01-18-20-11-01.bag b/mo0b6mh9/episode_rosbags/episode_19_2025-01-18-20-11-01.bag
new file mode 100644
index 0000000000000000000000000000000000000000..9c4f63eebcdfced4899c306fad680513722dbace
--- /dev/null
+++ b/mo0b6mh9/episode_rosbags/episode_19_2025-01-18-20-11-01.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc4b673bba2ffdca8e03c518067c33265a755ecd2b3ca0ebff1b4eb932668b7a
+size 1411832148
diff --git a/mo0b6mh9/episode_rosbags/episode_1_2025-01-18-19-56-25.bag b/mo0b6mh9/episode_rosbags/episode_1_2025-01-18-19-56-25.bag
new file mode 100644
index 0000000000000000000000000000000000000000..dd84109094cf0a4895ca951ad8008be234cb4694
--- /dev/null
+++ b/mo0b6mh9/episode_rosbags/episode_1_2025-01-18-19-56-25.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67cfd96c67efbb8afaaac412b52f309e340f77bf4ab56658f5d3ca3a677b3581
+size 1412687206
diff --git a/mo0b6mh9/episode_rosbags/episode_3_2025-01-18-19-57-53.bag b/mo0b6mh9/episode_rosbags/episode_3_2025-01-18-19-57-53.bag
new file mode 100644
index 0000000000000000000000000000000000000000..86e73e95be0ba0ad8b09838f75ef14f6e66f0bb8
--- /dev/null
+++ b/mo0b6mh9/episode_rosbags/episode_3_2025-01-18-19-57-53.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d550a819c56473514c17693a5f3a57f06c86c3140e241541f7c5f5e1779f8062
+size 1414307005
diff --git a/mo0b6mh9/episode_rosbags/episode_6_2025-01-18-20-00-08.bag b/mo0b6mh9/episode_rosbags/episode_6_2025-01-18-20-00-08.bag
new file mode 100644
index 0000000000000000000000000000000000000000..8caaef0c80c413c2c592abaa0e784a1eae9eb1f1
--- /dev/null
+++ b/mo0b6mh9/episode_rosbags/episode_6_2025-01-18-20-00-08.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdc359e74e1bc90a2b5f2facbfe06f3e85420004e7afc3bd89c646b803ab7e10
+size 1413537069
diff --git a/mo0b6mh9/episode_rosbags/episode_7_2025-01-18-20-00-52.bag b/mo0b6mh9/episode_rosbags/episode_7_2025-01-18-20-00-52.bag
new file mode 100644
index 0000000000000000000000000000000000000000..b22eab612ab9cd129f8c2430662d6de829abde09
--- /dev/null
+++ b/mo0b6mh9/episode_rosbags/episode_7_2025-01-18-20-00-52.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35478e91680b4f154fe1231b12e4328ca343ab9500f73ab3934d0d5548141256
+size 1412375315
diff --git a/q1ti958r/.hydra/config.yaml b/q1ti958r/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..dd7a0d9043cd058869b33cb503156d329d779fab
--- /dev/null
+++ b/q1ti958r/.hydra/config.yaml
@@ -0,0 +1,352 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 20
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '3465_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      crop_distractors_tlhw: null
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+      crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw}
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/q1ti958r/.hydra/hydra.yaml b/q1ti958r/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7fbd069f816af13753c30e1086c85eab5ad0c51c
--- /dev/null
+++ b/q1ti958r/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/q1ti958r/.hydra/overrides.yaml b/q1ti958r/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/q1ti958r/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/q1ti958r/episode_rosbags/aligned_depth_to_color_K.npy b/q1ti958r/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/q1ti958r/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/q1ti958r/episode_rosbags/color_K.npy b/q1ti958r/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/q1ti958r/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/q1ti958r/episode_rosbags/episode_12_2025-01-18-18-59-17.bag b/q1ti958r/episode_rosbags/episode_12_2025-01-18-18-59-17.bag
new file mode 100644
index 0000000000000000000000000000000000000000..494d271c7176668ecbc9c220d724b6a907762a9d
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_12_2025-01-18-18-59-17.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a60cd91450bb43d9c32f2721615c5f3c3d08239fac8d772801e640c047a058b9
+size 1412510226
diff --git a/q1ti958r/episode_rosbags/episode_16_2025-01-18-19-02-54.bag b/q1ti958r/episode_rosbags/episode_16_2025-01-18-19-02-54.bag
new file mode 100644
index 0000000000000000000000000000000000000000..086fc9d806a7723154a8a74e28b0a0767d987fdd
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_16_2025-01-18-19-02-54.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:493736a46d0a9693291a3ce776160b6835435b5c919a30a0e315eed8b9376941
+size 1410359279
diff --git a/q1ti958r/episode_rosbags/episode_18_2025-01-18-19-05-03.bag b/q1ti958r/episode_rosbags/episode_18_2025-01-18-19-05-03.bag
new file mode 100644
index 0000000000000000000000000000000000000000..adf16956959ac3eba506d4cccc7aa05d07839e24
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_18_2025-01-18-19-05-03.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7bc2c306a06803b785ff211a7dc6c6e10b7646c0ee16255ebd38d79a75c309d
+size 1415993987
diff --git a/q1ti958r/episode_rosbags/episode_2_2025-01-18-18-51-59.bag b/q1ti958r/episode_rosbags/episode_2_2025-01-18-18-51-59.bag
new file mode 100644
index 0000000000000000000000000000000000000000..d8916a04efe65f0611482ef6dcad46aeedae36d6
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_2_2025-01-18-18-51-59.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3dc9ede134fb2f192b223e334e2b71a90777b6fdadd21d31b2d8564218bb373
+size 1414254595
diff --git a/q1ti958r/episode_rosbags/episode_3_2025-01-18-18-52-46.bag b/q1ti958r/episode_rosbags/episode_3_2025-01-18-18-52-46.bag
new file mode 100644
index 0000000000000000000000000000000000000000..4982128c18ef2fec893630f2e30d61873ca3f333
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_3_2025-01-18-18-52-46.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dca6793b982f3bb6c6f58f22f066c92aea1e541d428045978d89cc92cb61e9ea
+size 1412215256
diff --git a/q1ti958r/episode_rosbags/episode_4_2025-01-18-18-53-26.bag b/q1ti958r/episode_rosbags/episode_4_2025-01-18-18-53-26.bag
new file mode 100644
index 0000000000000000000000000000000000000000..563f6161fbb7f86ea100f80df2980bb3c51a05c4
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_4_2025-01-18-18-53-26.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:371de18810fd9df3b16e84b1933dc7e785adf694d0b7fb6f9cb55e26f5afd0cc
+size 1406591734
diff --git a/q1ti958r/episode_rosbags/episode_5_2025-01-18-18-54-09.bag b/q1ti958r/episode_rosbags/episode_5_2025-01-18-18-54-09.bag
new file mode 100644
index 0000000000000000000000000000000000000000..169b8bbec5dc145709e8b072a258055113a4d429
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_5_2025-01-18-18-54-09.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d3ae3449ce664275dcaf8e254f9ba9c6cc501e6566e07fb8faaff7c76a4c7cd
+size 1409661126
diff --git a/q1ti958r/episode_rosbags/episode_9_2025-01-18-18-57-05.bag b/q1ti958r/episode_rosbags/episode_9_2025-01-18-18-57-05.bag
new file mode 100644
index 0000000000000000000000000000000000000000..74fe41daa4297169aa377455a0c3b8667ea2a03d
--- /dev/null
+++ b/q1ti958r/episode_rosbags/episode_9_2025-01-18-18-57-05.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d936247f4a68c2f07c9449a4fd1849eec6831cf2d57a12bd59959b288d9df4e
+size 1414220823
diff --git a/q1ti958r/eval_video/0_eval.mp4 b/q1ti958r/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4b88b028e3d9b6311b8b139bf84e652f83215879
--- /dev/null
+++ b/q1ti958r/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c4b549c276868919c0f491a0b92fc7d64d18307e5b08b0aa71ecf98ae9c0395
+size 1183093
diff --git a/q1ti958r/eval_video/10_eval.mp4 b/q1ti958r/eval_video/10_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..447afece1c17d9221efe20dad08fa7761c93326b
--- /dev/null
+++ b/q1ti958r/eval_video/10_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f1bedc7d64acfd933487bd4ba577ee603bd72ff2ee29025f80da331a2adcbec
+size 1161488
diff --git a/q1ti958r/eval_video/11_eval.mp4 b/q1ti958r/eval_video/11_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1d0a6932e280c93bfee5a278405ea411128c4060
--- /dev/null
+++ b/q1ti958r/eval_video/11_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:603d199abb1a8aa26ee32b8e11cded12fb329ffb576d7926db6d9b72da16d453
+size 1096372
diff --git a/q1ti958r/eval_video/12_eval.mp4 b/q1ti958r/eval_video/12_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..49ba6454cb2834d67f8339a69622f581e94b63e0
--- /dev/null
+++ b/q1ti958r/eval_video/12_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:815bf7acfa2781cd7af56580908e50b744e833fb44d89be5ad368026fd0ae034
+size 1093953
diff --git a/q1ti958r/eval_video/13_eval.mp4 b/q1ti958r/eval_video/13_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6e5bad01996e0a43db8e991b1b4d4fde83b97635
--- /dev/null
+++ b/q1ti958r/eval_video/13_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be54cbdf6a2c74d5872d0943711b9fe5c08511a91299f75ff586a67d760c839e
+size 1426112
diff --git a/q1ti958r/eval_video/14_eval.mp4 b/q1ti958r/eval_video/14_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0ef4c71c1548ed7e577939201de0a93f24f6002d
--- /dev/null
+++ b/q1ti958r/eval_video/14_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015a814176a13d9d5eb6c8534c22d3b4b7ece0cf09da24ca48842f587c3ce754
+size 1179201
diff --git a/q1ti958r/eval_video/15_eval.mp4 b/q1ti958r/eval_video/15_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6cc0d5e45e2580bb9f0c7222c7e5a9c0f8a712e9
--- /dev/null
+++ b/q1ti958r/eval_video/15_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fddbe9cae0cebc69c5e804deeddb6b643f5c55a27d5b00e57a3401fb2cc80b20
+size 1525087
diff --git a/q1ti958r/eval_video/16_eval.mp4 b/q1ti958r/eval_video/16_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f4a388be2be4f1e2116f5a6965d569c41594e046
--- /dev/null
+++ b/q1ti958r/eval_video/16_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b69f164b68c214c27944f0cc4822c6a56de1346eff9f1b034e2b58c05e0773e
+size 1453552
diff --git a/q1ti958r/eval_video/17_eval.mp4 b/q1ti958r/eval_video/17_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f818515be95d24ce6761492f48d5d96007b5ef81
--- /dev/null
+++ b/q1ti958r/eval_video/17_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97651cd5baa14e04e160fdb97e27e86d4c5d717eef4b18d53243b5d5e7de59e9
+size 1165331
diff --git a/q1ti958r/eval_video/18_eval.mp4 b/q1ti958r/eval_video/18_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0eaef10c1000da3d0b99cbbba973d517afdf4179
--- /dev/null
+++ b/q1ti958r/eval_video/18_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b8fe7251cc74d7e3aaab94983c785f28362553c3e41f0961ef1c5ab149d7bd1
+size 1457605
diff --git a/q1ti958r/eval_video/19_eval.mp4 b/q1ti958r/eval_video/19_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..14d7f80e8fde7cc5b749ee303e9a9b7352318233
--- /dev/null
+++ b/q1ti958r/eval_video/19_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c92e633fda071abe0bf478353c4246277cc742804c3416ec17efc035ad2372d
+size 1584151
diff --git a/q1ti958r/eval_video/1_eval.mp4 b/q1ti958r/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9ef044ff73a7312ba5c5720681f4a0cbec1b91a2
--- /dev/null
+++ b/q1ti958r/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c73c1e8035e9955d130d826b1f6ed12daebb02196b107dbcbd450af0ab8616
+size 1178950
diff --git a/q1ti958r/eval_video/2_eval.mp4 b/q1ti958r/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ada7cc3f6f4e61cfee2b9bb1da923fa2def01d40
--- /dev/null
+++ b/q1ti958r/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d91598e5051835f677e846bba3b3b20182672d199ba7b980b67bae56e49ef01d
+size 1178401
diff --git a/q1ti958r/eval_video/3_eval.mp4 b/q1ti958r/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..baaa17ea366c5a3ec61225c1cc6ec06d220b1936
--- /dev/null
+++ b/q1ti958r/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6706618b69c2f852f27029cd3f2623d90627580203ae03adeb4efded570bea41
+size 1165610
diff --git a/q1ti958r/eval_video/4_eval.mp4 b/q1ti958r/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d1661d164433d6537313e819ad1b4526177e2540
--- /dev/null
+++ b/q1ti958r/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae4f4aa5127f311e839c042eb90d298ecf8855e044de2406195f4cee84bcfbff
+size 1155941
diff --git a/q1ti958r/eval_video/5_eval.mp4 b/q1ti958r/eval_video/5_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..254272d117860b6c81aefaa5b9da9b1c14f0e683
--- /dev/null
+++ b/q1ti958r/eval_video/5_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15c189fb02c615acea539d0a97915ccf4d44cd14689c04e50ee7f901c5ea5f2d
+size 1176984
diff --git a/q1ti958r/eval_video/6_eval.mp4 b/q1ti958r/eval_video/6_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4e50d0cf2fd2f15279ede6d1ddee6483c75b16cb
--- /dev/null
+++ b/q1ti958r/eval_video/6_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cccf0749ac898e59a2f97b1f9f9697afc18dd70cb85e8188a0ac5dae96fc2b46
+size 1121447
diff --git a/q1ti958r/eval_video/7_eval.mp4 b/q1ti958r/eval_video/7_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a9a81fccdb7c3ef2402aa50b76da8e0f5391c5be
--- /dev/null
+++ b/q1ti958r/eval_video/7_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21e28c4cb1d06fc94de3ad9d2764e5465abc5297bca213abd8a93ff403f53385
+size 1091786
diff --git a/q1ti958r/eval_video/8_eval.mp4 b/q1ti958r/eval_video/8_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..b739b4c405042cc13aa2287dcbc64b79171eebe8
--- /dev/null
+++ b/q1ti958r/eval_video/8_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7ac84718a1584e3ff200aae1c4f78cb649f64896fcf503ac4175073d3e090ce
+size 1305681
diff --git a/q1ti958r/eval_video/9_eval.mp4 b/q1ti958r/eval_video/9_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a5d045fcdc6d5b4027f2c76af5aa4c9c5b297ba5
--- /dev/null
+++ b/q1ti958r/eval_video/9_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1569ddaa0ccfb738c03bdf185527443c7f965b1b7aefa2a7074d57b151b7771
+size 1347834
diff --git a/q1ti958r/tb/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0 b/q1ti958r/tb/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
new file mode 100644
index 0000000000000000000000000000000000000000..38d53f7374f0dd60afcc375dbb238930d3339995
--- /dev/null
+++ b/q1ti958r/tb/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46e34ce0f59401c91b4166c175ad819efe0a83af2b06f3f78f3ff5ba4ae06e88
+size 22784
diff --git a/q1ti958r/wandb/debug-internal.log b/q1ti958r/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..d7a2edf2da28ae0bc73b40228f8e143a559d5541
--- /dev/null
+++ b/q1ti958r/wandb/debug-internal.log
@@ -0,0 +1,2295 @@
+2025-01-18 18:48:45,741 INFO    StreamThr :2553803 [internal.py:wandb_internal():85] W&B internal server running at pid: 2553803, started at: 2025-01-18 18:48:45.740809
+2025-01-18 18:48:45,742 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status
+2025-01-18 18:48:45,754 INFO    WriterThread:2553803 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb
+2025-01-18 18:48:45,755 DEBUG   SenderThread:2553803 [sender.py:send():379] send: header
+2025-01-18 18:48:45,764 DEBUG   SenderThread:2553803 [sender.py:send():379] send: run
+2025-01-18 18:48:46,002 INFO    SenderThread:2553803 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files
+2025-01-18 18:48:46,002 INFO    SenderThread:2553803 [sender.py:_start_run_threads():1188] run started: q1ti958r with start time 1737244125.74124
+2025-01-18 18:48:46,019 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: check_version
+2025-01-18 18:48:46,019 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: check_version
+2025-01-18 18:48:46,095 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: run_start
+2025-01-18 18:48:46,099 DEBUG   HandlerThread:2553803 [system_info.py:__init__():26] System info init
+2025-01-18 18:48:46,099 DEBUG   HandlerThread:2553803 [system_info.py:__init__():41] System info init done
+2025-01-18 18:48:46,100 INFO    HandlerThread:2553803 [system_monitor.py:start():194] Starting system monitor
+2025-01-18 18:48:46,100 INFO    SystemMonitor:2553803 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-18 18:48:46,100 INFO    HandlerThread:2553803 [system_monitor.py:probe():214] Collecting system info
+2025-01-18 18:48:46,100 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started cpu monitoring
+2025-01-18 18:48:46,100 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started disk monitoring
+2025-01-18 18:48:46,102 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started gpu monitoring
+2025-01-18 18:48:46,102 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started memory monitoring
+2025-01-18 18:48:46,102 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started network monitoring
+2025-01-18 18:48:46,112 DEBUG   HandlerThread:2553803 [system_info.py:probe():152] Probing system
+2025-01-18 18:48:46,114 DEBUG   HandlerThread:2553803 [system_info.py:_probe_git():137] Probing git
+2025-01-18 18:48:46,119 DEBUG   HandlerThread:2553803 [system_info.py:_probe_git():145] Probing git done
+2025-01-18 18:48:46,119 DEBUG   HandlerThread:2553803 [system_info.py:probe():200] Probing system done
+2025-01-18 18:48:46,119 DEBUG   HandlerThread:2553803 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-18T23:48:46.112659', 'startedAt': '2025-01-18T23:48:45.733511', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3752.0185, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.938, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4548.31, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 498.2482566833496}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-18 18:48:46,120 INFO    HandlerThread:2553803 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-18 18:48:46,120 INFO    HandlerThread:2553803 [system_monitor.py:probe():227] Publishing system info
+2025-01-18 18:48:46,120 DEBUG   HandlerThread:2553803 [system_info.py:_save_code():44] Saving code
+2025-01-18 18:48:46,121 DEBUG   HandlerThread:2553803 [system_info.py:_save_code():67] Saving code done
+2025-01-18 18:48:46,121 DEBUG   HandlerThread:2553803 [system_info.py:_save_patches():84] Saving git patches
+2025-01-18 18:48:46,218 DEBUG   HandlerThread:2553803 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-18 18:48:46,219 INFO    HandlerThread:2553803 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-18 18:48:46,226 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:48:46,226 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-18 18:48:46,226 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-18 18:48:46,226 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-18 18:48:46,227 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now
+2025-01-18 18:48:46,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-18 18:48:46,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: python_packages
+2025-01-18 18:48:46,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:48:46,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:48:46,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:46,475 INFO    HandlerThread:2553803 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/tb"
+  save: true
+}
+control {
+  relay_id: "3ad0bdfad9e7418390329a14bb5b86ea"
+}
+_info {
+  stream_id: "q1ti958r"
+}
+
+2025-01-18 18:48:46,475 DEBUG   HandlerThread:2553803 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-18 18:48:46,492 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/7eqh4ohy-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
+2025-01-18 18:48:46,492 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/4vr9lobk-code/FISH/eval_robot.py
+2025-01-18 18:48:46,497 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/o1efaon9-wandb-metadata.json
+2025-01-18 18:48:46,522 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/zsz3gief-diff.patch
+2025-01-18 18:48:46,559 DEBUG   SenderThread:2553803 [sender.py:send():379] send: telemetry
+2025-01-18 18:48:46,560 DEBUG   SenderThread:2553803 [sender.py:send():379] send: tbrecord
+2025-01-18 18:48:46,560 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:48:46,560 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0 with policy live
+2025-01-18 18:48:46,744 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/k6c5de3k-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH
+2025-01-18 18:48:47,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:48,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:49,003 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:48:49,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:50,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:51,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:48:51,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:52,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:53,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:54,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:55,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:56,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:48:56,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:57,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:58,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:59,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:00,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:01,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:01,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:01,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:01,564 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:02,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:03,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:04,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:05,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:06,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:07,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:07,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:08,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:09,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:10,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:11,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:12,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:12,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:13,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:14,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:15,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:16,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:16,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:16,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:17,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:17,625 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:18,011 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 18:49:18,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:19,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:20,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:21,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:22,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:23,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:23,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:24,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:25,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:26,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:27,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:28,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:29,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:29,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:30,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:31,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:31,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:32,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:33,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:34,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:34,557 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:35,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:36,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:37,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:38,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:39,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:40,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:40,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:41,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:42,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:43,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:44,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:45,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:45,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:46,103 DEBUG   SystemMonitor:2553803 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-18 18:49:46,103 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:49:46,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:46,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:46,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:47,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:48,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:49,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:50,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:50,549 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:51,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:52,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:53,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:54,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:55,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:56,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:56,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:57,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:58,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:59,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:00,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:01,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:01,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:01,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:01,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:02,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:03,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:04,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:05,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:06,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:07,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:07,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:08,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:09,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:10,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:11,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:12,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:13,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:13,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:14,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:15,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:16,104 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:50:16,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:16,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:16,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:17,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:18,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:19,013 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:19,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:20,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:21,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:22,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:23,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:24,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:24,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:24,759 DEBUG   SenderThread:2553803 [sender.py:send():379] send: config
+2025-01-18 18:50:25,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:26,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:27,031 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:27,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:28,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:29,032 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:29,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:30,292 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:30,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:31,033 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:31,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:31,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:32,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:33,033 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:33,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:34,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:35,034 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:35,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:36,149 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:36,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:37,035 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:37,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:38,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:39,035 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:39,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:40,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:41,036 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:41,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:41,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:42,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:43,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:44,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:45,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:46,104 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:50:46,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:46,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:46,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:46,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:47,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:48,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:49,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:50,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:51,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:52,277 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:52,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:53,039 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:53,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:54,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:55,040 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:55,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:56,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:57,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:57,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:58,041 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 18:50:58,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:59,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:00,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:01,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:01,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:01,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:02,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:03,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:03,554 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:04,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:05,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:06,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:06,808 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:51:06,809 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4 with policy now
+2025-01-18 18:51:06,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:51:06,810 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:51:06,810 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:51:06,810 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:51:07,043 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:51:07,074 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/t587w10y-media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 18:51:07,386 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/neoboxbz-media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 18:51:07,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:08,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:08,817 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:09,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:09,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:10,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:11,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:12,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:13,045 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:13,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:14,073 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:14,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:15,046 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:15,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:16,047 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:51:16,111 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:51:16,305 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fbld0vc8-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:51:16,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:16,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:16,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:17,047 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:17,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:18,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:19,048 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:19,357 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:19,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:20,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:21,049 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:21,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:22,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:23,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:24,377 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:24,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:25,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:26,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:27,051 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:27,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:28,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:29,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:29,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:30,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:31,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:31,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:32,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:33,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:34,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:34,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:35,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:36,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:37,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:38,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:39,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:40,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:40,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:41,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:42,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:43,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:44,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:45,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:46,105 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:51:46,106 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:46,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:46,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:46,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:47,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:47,799 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:51:47,799 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4 with policy now
+2025-01-18 18:51:47,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:51:47,800 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:51:47,800 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:51:47,801 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:51:48,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:51:48,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 18:51:48,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:51:48,059 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/u93kbo3q-media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 18:51:48,338 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/t0g3xcz1-media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 18:51:48,503 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:49,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:49,504 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:50,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:50,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:51,338 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:51,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:52,510 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:53,059 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:53,509 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:54,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:55,060 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:55,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:55,990 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:51:55,990 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:51:55,991 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:51:56,061 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:51:56,061 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:51:56,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:56,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:57,061 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:57,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:58,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:59,062 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:59,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:00,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:01,063 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:01,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:01,478 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:01,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:01,615 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:02,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:03,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:04,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:05,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:06,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:07,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:07,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:08,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:09,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:10,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:11,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:12,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:12,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:13,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:14,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:15,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:16,106 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:52:16,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:16,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:16,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:17,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:17,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:18,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:19,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:20,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:21,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:22,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:23,070 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:23,257 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:23,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:24,575 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:25,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:26,575 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:27,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:27,896 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:52:27,896 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4 with policy now
+2025-01-18 18:52:27,899 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:52:27,899 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:52:27,899 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:52:27,900 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:52:28,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:52:28,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 18:52:28,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:52:28,149 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/e8z32mvu-media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 18:52:28,431 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/6l74dbyw-media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 18:52:28,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:28,918 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:29,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:29,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:30,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:30,586 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:31,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:31,477 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:31,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:32,581 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:33,073 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:33,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:34,172 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:34,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:35,074 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:35,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:36,582 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:37,584 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:38,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:39,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:39,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:40,584 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:41,598 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:42,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:42,824 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:52:42,825 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:52:42,825 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:52:43,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:43,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:52:43,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:52:43,338 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ey0x170e-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:52:43,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:44,591 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:45,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:45,361 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:45,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:46,106 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:52:46,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:46,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:46,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:47,078 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:47,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:48,586 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:49,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:50,534 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:50,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:51,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:52,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:53,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:54,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:55,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:56,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:56,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:57,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:58,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:59,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:00,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:01,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:01,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:01,553 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:01,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:02,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:03,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:04,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:05,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:06,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:07,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:07,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:08,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:09,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:10,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:11,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:12,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:12,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:13,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:14,497 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:53:14,497 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4 with policy now
+2025-01-18 18:53:14,503 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:53:14,504 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:53:14,504 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:53:14,505 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:53:14,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:14,780 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/aqhxa1wy-media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:53:15,391 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/2467bf5q-media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 18:53:15,599 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:16,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:16,107 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:53:16,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:16,480 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:16,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:17,089 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:17,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:17,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:18,601 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:19,089 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:19,602 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:20,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:21,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:22,613 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:22,875 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:53:22,875 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:53:22,875 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:22,876 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:53:23,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:23,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:53:23,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:53:23,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:24,605 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:25,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:25,611 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:26,616 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:27,092 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:27,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:28,556 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:28,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:29,093 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:29,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:30,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:31,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:31,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:31,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:32,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:33,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:33,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:34,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:35,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:36,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:37,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:38,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:39,096 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:39,337 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:39,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:40,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:41,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:42,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:43,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:44,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:44,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:45,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:46,107 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:53:46,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:46,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:46,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:47,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:48,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:49,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:49,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:50,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:51,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:52,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:53,610 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:54,617 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:55,050 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:55,102 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:55,625 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:55,697 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:53:55,697 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:53:55,698 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4 with policy now
+2025-01-18 18:53:55,698 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:53:55,699 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:53:55,699 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:53:55,969 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/eylvpie0-media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 18:53:56,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:53:56,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 18:53:56,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:53:56,633 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:56,645 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/aqskltpv-media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 18:53:57,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:57,629 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:58,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:58,634 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:59,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:00,269 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:00,637 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:01,104 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:01,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:01,496 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:01,632 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:02,632 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:03,105 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:03,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:04,640 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:05,447 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:05,645 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:05,927 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:54:05,927 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:54:05,928 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:54:06,106 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:54:06,106 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:54:06,360 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/aczj1xtw-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:54:06,641 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:07,107 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:07,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:08,654 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:09,107 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:09,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:10,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:10,782 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:11,108 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:11,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:12,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:13,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:14,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:15,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:16,108 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:54:16,108 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:16,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:16,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:16,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:17,110 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:17,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:18,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:19,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:20,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:21,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:21,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:22,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:23,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:24,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:25,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:26,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:26,645 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:27,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:28,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:29,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:30,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:31,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:31,476 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:31,598 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:31,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:32,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:33,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:34,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:35,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:36,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:37,307 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:37,654 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:37,918 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:54:37,918 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4 with policy now
+2025-01-18 18:54:37,919 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:54:37,919 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:54:37,919 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:54:37,922 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:54:38,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:54:38,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 18:54:38,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:54:38,208 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/i4z1pkjl-media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 18:54:38,490 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/tal85z4g-media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 18:54:38,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:39,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:39,648 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:40,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:40,654 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:41,663 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:42,510 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:42,672 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:43,119 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:43,664 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:44,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:45,120 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:45,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:46,108 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:54:46,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:46,493 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:46,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:47,619 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:47,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:48,672 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:49,684 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:49,982 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:54:49,982 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:54:49,982 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:54:50,122 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:54:50,122 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:54:50,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:51,122 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:51,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:52,672 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:53,123 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:53,512 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:53,671 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:54,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:55,124 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:55,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:56,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:57,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:58,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:59,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:59,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:00,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:01,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:01,480 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:01,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:02,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:03,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:04,556 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:04,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:05,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:06,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:07,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:08,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:09,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:10,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:10,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:11,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:12,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:13,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:14,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:15,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:15,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:16,108 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:55:16,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:16,479 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:16,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:17,132 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:17,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:18,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:19,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:20,543 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:20,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:21,676 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:21,988 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:55:21,988 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4 with policy now
+2025-01-18 18:55:21,991 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:55:21,991 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:55:21,991 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:55:21,992 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:55:22,134 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:55:22,134 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 18:55:22,135 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:55:22,251 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/9ibw7cch-media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 18:55:22,546 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/tbylwsyz-media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 18:55:22,674 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:23,135 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:23,673 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:24,135 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:24,689 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:25,554 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:25,681 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:26,685 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:27,136 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:27,680 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:28,677 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:29,137 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:29,686 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:30,690 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:31,029 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:55:31,030 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:55:31,030 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:31,030 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:55:31,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:31,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:55:31,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:55:31,394 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/uh15q6l5-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:55:31,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:31,491 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:31,690 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:32,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:33,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:33,692 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:34,690 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:35,139 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:35,691 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:36,466 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:36,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:37,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:38,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:39,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:40,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:41,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:41,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:42,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:43,142 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:43,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:44,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:45,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:46,109 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:55:46,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:46,480 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:46,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:46,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:47,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:48,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:49,145 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:49,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:50,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:51,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:52,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:52,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:53,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:54,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:55,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:56,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:57,507 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:57,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:58,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:59,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:00,696 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:01,149 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:01,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:01,495 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:01,700 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:02,362 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:56:02,362 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4 with policy now
+2025-01-18 18:56:02,371 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:56:02,379 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:56:02,379 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:56:02,379 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:56:02,627 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/sluts8c9-media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 18:56:02,703 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:56:03,431 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:03,465 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/cw47cria-media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 18:56:03,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:04,722 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:05,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:06,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:07,151 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:07,723 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:08,535 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:08,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:09,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:10,720 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:11,153 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:11,744 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:12,080 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:56:12,080 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:56:12,081 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:56:12,153 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:56:12,154 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:56:12,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:13,154 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:13,677 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:13,742 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:14,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:15,155 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:15,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:16,109 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:56:16,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:16,489 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:16,737 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:17,155 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:17,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:18,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:19,320 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:19,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:20,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:21,157 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:21,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:22,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:23,158 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:23,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:24,517 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:24,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:25,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:26,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:27,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:28,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:29,160 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:29,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:30,013 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:30,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:31,161 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:31,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:31,484 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:31,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:32,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:33,162 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:33,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:34,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:35,225 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:35,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:36,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:37,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:38,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:39,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:40,518 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:40,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:41,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:42,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:43,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:44,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:45,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:46,110 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:56:46,111 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:46,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:46,484 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:46,745 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:47,167 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:47,653 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:56:47,653 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4 with policy now
+2025-01-18 18:56:47,663 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:56:47,664 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:56:47,664 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:56:47,664 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:56:47,738 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:47,924 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fjt46xbn-media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 18:56:48,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:56:48,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 18:56:48,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:56:48,477 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/y14eyeb0-media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 18:56:48,770 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:49,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:49,769 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:50,763 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:51,137 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:51,761 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:52,774 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:53,169 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:53,767 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:54,782 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:55,170 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:55,767 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:56,145 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:56,766 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:57,781 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:58,778 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:59,783 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:00,783 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:01,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:01,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:01,522 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:01,785 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:02,138 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:57:02,138 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:57:02,139 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:57:02,173 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:57:02,173 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:57:02,472 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/uczda84y-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:57:02,790 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:03,173 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:03,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:04,813 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:05,174 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:05,788 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:06,527 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:06,817 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:07,174 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:07,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:08,818 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:09,811 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:10,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:11,176 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:11,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:12,523 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:12,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:13,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:14,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:15,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:16,110 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:57:16,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:16,485 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:16,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:17,178 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:17,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:17,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:18,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:19,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:20,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:21,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:22,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:23,523 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:23,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:24,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:25,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:26,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:27,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:28,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:29,523 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:29,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:30,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:31,485 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:31,485 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:31,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:32,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:33,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:34,540 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:34,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:35,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:36,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:37,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:38,567 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:57:38,568 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4 with policy now
+2025-01-18 18:57:38,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:57:38,594 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:57:38,594 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:57:38,595 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:57:38,822 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:38,851 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/34mx1zxu-media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:57:39,516 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/1qdkvnax-media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 18:57:39,628 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:39,833 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:40,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:40,843 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:41,833 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:42,864 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:43,188 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:43,854 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:44,855 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:45,253 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:45,856 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:46,111 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:57:46,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:46,497 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:46,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:47,190 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:47,857 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:48,852 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:49,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:49,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:57:49,197 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:57:49,197 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:57:49,198 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:57:49,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:50,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:57:50,701 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:50,879 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:51,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:51,883 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:52,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:53,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:54,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:55,193 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:55,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:56,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:56,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:57,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:58,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:59,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:00,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:01,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:01,488 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:01,631 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:01,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:02,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:03,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:04,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:05,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:06,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:07,552 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:07,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:08,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:09,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:10,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:11,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:12,554 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:12,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:13,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:14,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:15,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:16,111 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:58:16,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:16,513 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:16,869 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:17,201 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:17,624 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:17,869 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:18,870 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:19,889 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:20,876 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:21,148 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:58:21,148 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4 with policy now
+2025-01-18 18:58:21,152 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:58:21,152 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:58:21,153 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:58:21,153 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:58:21,203 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:21,203 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:58:21,203 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 18:58:21,204 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:58:21,430 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ehpxwp7a-media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 18:58:21,767 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/w390kric-media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 18:58:21,901 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:22,889 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:23,204 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:23,558 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:23,896 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:24,891 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:25,891 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:26,909 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:27,205 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:27,941 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:28,920 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:28,942 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:29,206 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:29,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:30,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:31,207 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:58:31,249 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:58:31,250 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:58:31,250 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:58:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:31,513 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:31,928 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:32,207 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:58:32,208 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:58:32,473 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/coyho2su-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:58:32,912 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:33,208 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:33,949 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:34,305 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:34,920 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:35,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:36,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:37,209 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:37,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:38,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:39,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:40,117 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:40,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:41,211 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:41,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:42,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:43,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:44,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:45,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:45,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:46,112 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:58:46,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:46,499 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:46,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:47,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:48,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:49,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:50,596 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:50,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:51,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:52,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:53,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:54,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:55,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:56,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:56,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:57,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:58,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:59,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:00,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:01,219 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:01,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:01,499 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:01,595 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:01,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:02,917 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:03,220 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:03,823 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:59:03,823 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4 with policy now
+2025-01-18 18:59:03,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:59:03,826 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:03,826 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:03,827 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:03,922 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:04,107 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/awwdnehm-media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 18:59:04,220 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:04,220 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:04,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 18:59:04,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:59:04,604 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/4qny7a6b-media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 18:59:04,968 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:05,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:05,935 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:06,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:06,950 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:07,382 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:07,952 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:08,947 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:09,222 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:09,958 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:10,940 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:11,223 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:11,961 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:12,585 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:12,949 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:13,224 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:13,224 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:59:13,304 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:13,304 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:13,305 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:13,993 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:14,224 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:14,974 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:15,225 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:15,972 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:16,112 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:59:16,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:16,537 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:16,972 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:17,952 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:18,467 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:18,952 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:19,227 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:19,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:20,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:21,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:22,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:23,576 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:23,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:24,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:25,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:26,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:27,230 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:27,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:28,576 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:28,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:29,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:30,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:31,510 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:31,510 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:31,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:32,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:33,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:33,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:34,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:35,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:36,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:37,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:38,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:39,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:39,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:40,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:41,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:42,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:43,956 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:44,963 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:45,021 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:45,238 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:45,554 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:59:45,554 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4 with policy now
+2025-01-18 18:59:45,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:59:45,588 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:45,588 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:45,589 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:45,859 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ofnyy30v-media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 18:59:45,963 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:46,112 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:59:46,238 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:46,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:46,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 18:59:46,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:59:46,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:46,537 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:46,594 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ooipvqyr-media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 18:59:46,958 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:47,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:47,969 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:48,963 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:49,971 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:50,117 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:50,964 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:51,241 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:51,987 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:52,988 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:53,242 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:53,994 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:54,983 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:55,297 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:55,989 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:56,243 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:59:56,356 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:56,357 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:56,357 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:57,008 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:57,243 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:57,244 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:57,984 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:59,013 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:59,244 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:00,008 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:00,993 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:01,107 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:01,245 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:01,543 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:01,543 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:02,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:03,007 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:04,005 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:05,008 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:06,011 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:06,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:07,039 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:08,014 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:09,014 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:09,248 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:10,014 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:11,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:12,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:12,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:13,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:14,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:15,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:16,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:16,113 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:00:16,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:16,521 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:17,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:17,634 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:18,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:19,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:20,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:21,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:22,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:23,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:23,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:24,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:25,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:26,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:27,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:28,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:29,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:29,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:30,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:31,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:31,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:31,521 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:32,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:33,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:34,018 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:35,021 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:35,042 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:35,793 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:00:35,793 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4 with policy now
+2025-01-18 19:00:35,805 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:00:35,806 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:00:35,806 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:00:35,806 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:00:36,026 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:36,126 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/cnkxpgbz-media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:00:36,628 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/1c5mhm1s-media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:00:37,048 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:38,048 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:38,260 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:39,057 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:40,063 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:40,105 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:41,044 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:42,082 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:42,261 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:43,063 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:44,078 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:44,262 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:45,078 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:45,182 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:46,095 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:46,113 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:00:46,539 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:46,539 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:47,076 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:48,088 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:49,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:50,090 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:50,264 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:00:50,422 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:00:50,422 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:00:50,422 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:50,422 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:00:50,523 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/rssavz5p-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:00:51,122 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:51,265 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:00:52,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:52,266 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:53,107 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:54,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:55,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:56,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:56,268 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:56,314 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:57,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:58,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:59,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:00,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:01,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:01,533 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:01,534 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:01,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:02,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:03,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:04,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:05,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:06,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:07,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:07,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:08,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:09,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:10,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:11,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:12,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:12,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:13,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:14,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:15,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:16,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:16,114 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:01:16,532 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:16,533 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:17,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:17,691 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:18,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:19,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:20,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:21,147 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:22,120 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:22,279 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:22,644 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:01:22,644 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4 with policy now
+2025-01-18 19:01:22,658 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:01:22,659 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:01:22,659 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:01:22,659 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:01:22,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:22,949 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/1t8esl9o-media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:01:23,134 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:23,279 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:01:23,279 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:01:23,280 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:01:23,634 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/hfjrq3td-media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:01:24,127 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:24,280 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:25,158 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:26,164 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:27,138 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:28,170 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:28,281 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:28,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:29,138 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:30,144 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:30,282 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:01:30,472 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:01:30,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:01:30,474 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:01:31,158 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:31,283 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:01:31,540 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:31,540 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:32,162 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:32,283 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:33,163 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:34,067 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:34,163 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:35,185 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:36,162 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:36,285 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:37,170 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:38,175 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:39,196 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:39,628 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:40,193 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:41,186 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:42,222 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:43,234 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:44,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:44,288 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:45,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:45,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:46,114 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:01:46,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:46,289 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:46,538 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:46,538 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:47,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:48,188 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:49,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:50,188 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:50,659 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:51,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:52,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:53,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:54,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:55,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:56,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:56,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:57,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:58,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:59,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:00,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:01,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:01,538 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:01,539 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:01,692 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:02,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:03,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:04,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:05,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:06,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:07,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:07,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:08,191 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:09,191 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:10,229 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:11,211 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:11,927 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:02:11,927 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4 with policy now
+2025-01-18 19:02:11,937 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:02:11,938 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:02:11,938 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:02:11,938 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:02:12,200 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:12,246 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/juj99v43-media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:02:12,663 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/tq3lyu01-media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:02:12,987 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:13,218 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:14,233 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:14,301 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:15,227 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:16,115 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:02:16,224 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:16,585 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:16,586 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:17,227 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:18,274 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:18,302 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:18,777 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:19,231 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:20,289 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:20,303 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:21,267 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:22,268 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:23,279 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:24,295 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:24,649 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:25,293 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:26,263 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:27,290 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:28,291 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:29,283 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:30,282 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:30,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:31,286 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:31,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:31,583 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:32,283 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:33,285 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:34,317 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:35,314 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:35,709 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:36,292 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:37,316 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:38,315 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:39,308 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:40,311 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:40,719 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:41,341 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:42,320 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:43,333 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:44,371 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:45,357 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:46,115 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:02:46,117 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:46,378 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:46,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:46,588 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:47,438 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:48,389 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:49,394 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:50,316 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:02:50,378 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:50,566 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:02:50,566 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:02:50,566 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:02:51,304 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:51,316 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:02:51,403 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:52,317 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:52,391 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:53,384 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:54,317 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:54,408 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:55,410 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:56,318 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:56,407 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:56,529 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:57,447 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:58,413 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:59,437 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:00,446 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:01,446 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:01,610 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:01,611 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:01,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:02,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:03,466 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:04,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:05,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:06,323 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:06,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:07,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:07,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:08,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:09,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:10,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:11,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:12,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:12,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:13,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:14,326 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:14,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:15,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:16,116 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:03:16,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:16,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:16,574 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:17,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:17,678 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:18,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:19,445 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:20,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:21,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:22,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:23,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:23,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:24,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:25,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:26,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:27,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:28,332 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:28,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:28,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:29,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:30,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:31,449 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:31,581 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:31,582 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:32,334 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:32,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:32,549 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:03:32,549 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:03:32,549 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4 with policy now
+2025-01-18 19:03:32,550 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:03:32,550 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:03:32,550 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:03:32,861 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/9fl1agn2-media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:03:33,334 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:03:33,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:03:33,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:03:33,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:33,709 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/zm6gx6cm-media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:03:34,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:34,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:34,618 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:35,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:35,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:36,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:37,464 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:38,337 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:38,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:39,468 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:40,353 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:40,470 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:41,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:42,338 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:42,508 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:43,516 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:44,525 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:45,519 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:45,698 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:46,116 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:03:46,527 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:46,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:46,588 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:47,524 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:48,581 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:49,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:50,529 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:50,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:51,545 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:52,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:53,548 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:54,567 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:55,551 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:56,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:56,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:57,584 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:58,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:59,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:00,592 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:01,592 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:01,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:01,595 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:01,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:02,598 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:03,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:04,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:05,645 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:06,618 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:06,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:07,629 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:08,349 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:04:08,609 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fon0epps-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:04:08,637 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:08,659 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:04:08,659 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:04:08,660 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:04:09,350 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:04:09,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:10,350 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:10,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:11,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:11,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:12,351 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:12,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:13,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:14,352 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:14,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:15,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:16,116 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:04:16,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:16,579 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:16,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:17,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:17,714 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:18,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:19,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:20,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:21,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:22,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:23,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:23,704 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:24,356 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:24,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:25,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:26,357 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:26,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:27,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:28,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:28,704 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:29,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:30,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:31,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:31,579 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:31,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:32,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:33,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:33,704 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:34,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:35,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:36,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:37,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:38,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:39,428 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:39,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:40,636 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:40,978 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:04:40,978 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4 with policy now
+2025-01-18 19:04:40,999 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:04:40,999 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:04:40,999 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:04:41,000 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:04:41,262 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ruqorwuy-media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:04:41,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:04:41,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:04:41,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:04:41,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:41,690 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fxtr8vke-media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:04:42,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:42,710 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:43,365 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:43,641 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:44,656 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:44,684 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:45,686 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:46,117 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:04:46,366 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:46,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:46,584 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:46,651 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:47,664 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:48,666 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:49,662 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:50,053 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:50,368 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:50,662 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:51,653 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:52,660 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:53,700 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:54,691 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:55,682 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:55,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:56,710 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:57,675 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:58,707 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:59,372 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:04:59,721 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:59,721 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:04:59,721 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:04:59,722 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:05:00,372 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:00,372 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:05:00,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:01,165 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:01,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:01,609 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:01,712 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:02,373 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:02,776 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:03,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:04,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:05,744 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:06,375 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:06,533 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:06,724 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:07,737 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:08,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:09,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:10,737 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:11,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:11,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:12,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:13,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:14,378 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:14,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:15,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:16,117 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:05:16,379 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:16,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:16,580 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:16,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:17,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:17,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:18,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:19,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:20,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:21,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:22,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:23,717 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:23,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:24,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:25,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:26,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:27,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:28,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:29,717 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:29,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:30,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:31,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:31,580 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:31,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:32,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:33,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:34,721 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:34,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:35,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:36,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:37,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:38,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:39,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:40,063 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:40,393 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:40,670 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:05:40,670 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4 with policy now
+2025-01-18 19:05:40,679 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:05:40,679 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:05:40,679 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:05:40,679 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:05:40,734 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:40,981 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/5fk8t884-media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:05:41,390 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:05:41,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:05:41,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:05:41,743 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/c81m63pf-media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:05:41,761 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:42,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:42,811 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:43,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:43,804 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:44,770 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:45,305 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:45,782 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:46,118 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:05:46,393 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:46,601 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:46,601 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:46,785 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:47,754 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:48,759 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:49,764 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:50,315 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:50,394 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:50,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:05:50,759 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:50,780 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:05:50,780 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:05:50,781 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:05:51,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:05:51,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:05:51,771 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:52,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:52,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:53,789 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:54,396 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:54,788 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:55,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:55,961 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:56,397 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:56,820 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:57,796 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:58,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:59,830 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:00,813 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:01,613 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:01,613 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:01,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:01,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:02,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:03,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:04,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:05,825 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:06,734 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:06,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:07,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:08,402 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:08,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:09,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:10,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:11,734 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:11,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:12,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:13,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:14,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:15,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:16,118 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:06:16,596 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:16,596 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:16,735 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:16,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:17,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:18,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:19,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:20,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:21,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:21,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:22,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:23,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:24,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:25,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:26,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:26,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:27,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:28,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:29,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:30,412 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:30,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:31,596 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:31,596 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:31,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:31,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:32,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:33,885 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:34,657 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:06:34,658 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4 with policy now
+2025-01-18 19:06:34,675 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:06:34,675 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:34,675 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:34,676 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:34,813 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:34,977 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ea2nxp9j-media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:06:35,414 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:35,414 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:06:35,414 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:06:35,773 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/e8qe9gzx-media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:06:35,828 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:36,415 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:36,752 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:36,851 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:37,416 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:37,845 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:38,851 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:39,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:40,418 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:40,846 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:41,775 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:41,858 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:42,846 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:43,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:43,844 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:43,844 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:43,844 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:43,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:44,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:44,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:44,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:44,848 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:44,890 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-18 19:06:44,891 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: log_artifact
+2025-01-18 19:06:45,390 INFO    wandb-upload_0:2553803 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa2smyjth
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table
+2025-01-18 19:06:45,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:06:45,777 INFO    SenderThread:2553803 [sender.py:send_request_log_artifact():1525] logged artifact run-q1ti958r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODI5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAwNzM4NQ==', 'latestArtifact': None}}
+2025-01-18 19:06:45,777 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:06:45,777 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json with policy now
+2025-01-18 19:06:45,779 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:45,779 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,780 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,783 DEBUG   SenderThread:2553803 [sender.py:send():379] send: telemetry
+2025-01-18 19:06:45,784 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,785 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,785 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,786 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,787 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,788 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,789 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,789 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,789 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,790 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,791 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,791 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,792 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,792 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,796 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,796 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,797 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,797 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,800 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,800 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,801 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,801 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:45,873 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-18 19:06:45,873 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: log_artifact
+2025-01-18 19:06:45,917 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:06:45,965 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/3wfnb2p7-media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:06:46,224 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/w508hlfq-media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:06:46,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:46,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:46,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:46,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:06:46,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval
+2025-01-18 19:06:46,426 INFO    wandb-upload_3:2553803 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4cq89eih
+2025-01-18 19:06:46,597 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:46,608 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/jcv4qu0w-media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:06:46,726 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/xbeut6o6-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:46,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:46,871 INFO    SenderThread:2553803 [sender.py:send_request_log_artifact():1525] logged artifact run-q1ti958r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODI5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAwNzM4Nw==', 'latestArtifact': None}}
+2025-01-18 19:06:46,871 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:06:46,871 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json with policy now
+2025-01-18 19:06:46,871 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:46,871 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:46,872 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:46,872 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:46,872 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:06:46,873 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:47,127 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/zai78lqq-media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:06:47,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:47,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:48,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:48,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:49,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:50,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:51,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:52,747 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:52,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:53,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:54,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:55,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:56,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:57,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:58,747 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:58,851 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:59,813 DEBUG   SenderThread:2553803 [sender.py:send():379] send: exit
+2025-01-18 19:06:59,813 INFO    SenderThread:2553803 [sender.py:send_exit():586] handling exit code: 0
+2025-01-18 19:06:59,814 INFO    SenderThread:2553803 [sender.py:send_exit():588] handling runtime: 1093
+2025-01-18 19:06:59,814 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:59,814 INFO    SenderThread:2553803 [sender.py:send_exit():594] send defer
+2025-01-18 19:06:59,814 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,814 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-18 19:06:59,814 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 1
+2025-01-18 19:06:59,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-18 19:06:59,815 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 2
+2025-01-18 19:06:59,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-18 19:06:59,815 DEBUG   SystemMonitor:2553803 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-18 19:06:59,816 DEBUG   SystemMonitor:2553803 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-18 19:06:59,816 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined disk monitor
+2025-01-18 19:06:59,865 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-18 19:06:59,865 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined memory monitor
+2025-01-18 19:06:59,865 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined network monitor
+2025-01-18 19:06:59,865 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,865 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-18 19:06:59,866 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 3
+2025-01-18 19:06:59,866 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:06:59,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,866 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-18 19:06:59,866 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,866 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-18 19:06:59,866 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 4
+2025-01-18 19:06:59,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,866 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-18 19:07:00,428 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:07:02,430 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:07:06,852 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:06,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:07:06,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-18 19:07:06,853 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,854 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-18 19:07:06,854 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 5
+2025-01-18 19:07:06,854 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:06,854 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:06,854 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-18 19:07:06,854 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:07:06,855 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:07:06,855 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:07:06,856 DEBUG   SenderThread:2553803 [sender.py:send():379] send: summary
+2025-01-18 19:07:06,860 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:07:06,860 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,860 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-18 19:07:06,860 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 6
+2025-01-18 19:07:06,860 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:06,861 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-18 19:07:06,861 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,861 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-18 19:07:06,864 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:07:06,992 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 7
+2025-01-18 19:07:06,993 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:06,993 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-18 19:07:06,993 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,993 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-18 19:07:07,432 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:07:07,432 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 19:07:07,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:08,555 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 8
+2025-01-18 19:07:08,556 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:08,556 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:08,556 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-18 19:07:08,556 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:08,556 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-18 19:07:08,556 INFO    SenderThread:2553803 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-18 19:07:08,559 INFO    SenderThread:2553803 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-18 19:07:08,561 INFO    SenderThread:2553803 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-18 19:07:08,570 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 9
+2025-01-18 19:07:08,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:08,571 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-18 19:07:08,571 DEBUG   SenderThread:2553803 [sender.py:send():379] send: artifact
+2025-01-18 19:07:08,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:09,284 INFO    wandb-upload_0:2553803 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzx8vfrwt
+2025-01-18 19:07:09,284 INFO    wandb-upload_3:2553803 [upload_job.py:push():85] Skipped uploading /tmp/tmpsv0ptkxm/wandb-job.json
+2025-01-18 19:07:09,392 INFO    wandb-upload_2:2553803 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpcvsay24k
+2025-01-18 19:07:09,433 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:07:10,017 INFO    SenderThread:2553803 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODU1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTQwMTA5Mw==', 'versionIndex': 445}}}
+2025-01-18 19:07:10,017 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,017 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-18 19:07:10,017 INFO    SenderThread:2553803 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-18 19:07:10,434 INFO    SenderThread:2553803 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files
+2025-01-18 19:07:10,434 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml config.yaml
+2025-01-18 19:07:10,434 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json wandb-summary.json
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt requirements.txt
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json wandb-metadata.json
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log output.log
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0 events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch diff.patch
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4 media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4 media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4 media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4 media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4 media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4 media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4 media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4 media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4 media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4 media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4 media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4 media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4 media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4 media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4 media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4 media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4 media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4 media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4 media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4 media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 10
+2025-01-18 19:07:10,442 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:10,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:10,443 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-18 19:07:10,443 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,443 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-18 19:07:10,443 INFO    SenderThread:2553803 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-18 19:07:10,615 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt
+2025-01-18 19:07:10,636 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:07:10,644 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 19:07:10,688 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:07:10,816 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:10,816 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:10,889 INFO    Thread-11 (_thread_body):2553803 [sender.py:transition_state():614] send defer: 11
+2025-01-18 19:07:10,889 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:10,889 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-18 19:07:10,889 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,889 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-18 19:07:10,889 INFO    SenderThread:2553803 [file_pusher.py:join():175] waiting for file pusher
+2025-01-18 19:07:10,892 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 12
+2025-01-18 19:07:10,892 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:10,893 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-18 19:07:10,893 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,893 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-18 19:07:10,893 INFO    SenderThread:2553803 [file_stream.py:finish():601] file stream finish called
+2025-01-18 19:07:11,080 INFO    SenderThread:2553803 [file_stream.py:finish():605] file stream finish is done
+2025-01-18 19:07:11,080 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 13
+2025-01-18 19:07:11,080 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:11,080 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-18 19:07:11,080 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:11,080 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-18 19:07:11,081 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 14
+2025-01-18 19:07:11,081 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:11,081 DEBUG   SenderThread:2553803 [sender.py:send():379] send: final
+2025-01-18 19:07:11,081 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-18 19:07:11,081 DEBUG   SenderThread:2553803 [sender.py:send():379] send: footer
+2025-01-18 19:07:11,081 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:11,081 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-18 19:07:11,081 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:11,082 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:11,082 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: server_info
+2025-01-18 19:07:11,082 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:11,082 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: server_info
+2025-01-18 19:07:11,084 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-18 19:07:11,085 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-18 19:07:11,086 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:07:11,129 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:11,130 INFO    MainThread:2553803 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-18 19:07:11,130 INFO    MainThread:2553803 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-18 19:07:11,131 INFO    MainThread:2553803 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-18 19:07:11,131 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-18 19:07:11,131 INFO    HandlerThread:2553803 [handler.py:finish():882] shutting down handler
+2025-01-18 19:07:12,084 INFO    WriterThread:2553803 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb
+2025-01-18 19:07:12,130 INFO    SenderThread:2553803 [sender.py:finish():1615] shutting down sender
+2025-01-18 19:07:12,130 INFO    SenderThread:2553803 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-18 19:07:12,130 INFO    SenderThread:2553803 [file_pusher.py:join():175] waiting for file pusher
diff --git a/q1ti958r/wandb/debug.log b/q1ti958r/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..4885733bddd8dfce240f49cdbec9179dae64f66b
--- /dev/null
+++ b/q1ti958r/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Configure stats pid to 2553479
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/settings
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug.log
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug-internal.log
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_init.py:init():569] calling init triggers
+2025-01-18 18:48:45,736 INFO    MainThread:2553479 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r'}
+2025-01-18 18:48:45,736 INFO    MainThread:2553479 [wandb_init.py:init():619] starting backend
+2025-01-18 18:48:45,736 INFO    MainThread:2553479 [wandb_init.py:init():623] setting up manager
+2025-01-18 18:48:45,740 INFO    MainThread:2553479 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-18 18:48:45,741 INFO    MainThread:2553479 [wandb_init.py:init():631] backend started and connected
+2025-01-18 18:48:45,753 INFO    MainThread:2553479 [wandb_init.py:init():720] updated telemetry
+2025-01-18 18:48:45,762 INFO    MainThread:2553479 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-18 18:48:46,018 INFO    MainThread:2553479 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-18 18:48:46,090 INFO    MainThread:2553479 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-18 18:48:46,090 INFO    MainThread:2553479 [wandb_init.py:init():804] starting run threads in backend
+2025-01-18 18:48:46,471 INFO    MainThread:2553479 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-18 18:48:46,471 INFO    MainThread:2553479 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-18 18:48:46,472 INFO    MainThread:2553479 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-18 18:48:46,472 INFO    MainThread:2553479 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-18 18:48:46,474 INFO    MainThread:2553479 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-18 18:48:46,474 INFO    MainThread:2553479 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/tb, True
+2025-01-18 18:50:24,759 INFO    MainThread:2553479 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-18 19:07:12,132 WARNING MsgRouterThr:2553479 [router.py:message_loop():77] message_loop has been closed
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py
@@ -0,0 +1,606 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config), 
+				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				# self.cfg.agent.config.policy_cfg.input_shapes,
+				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3a72762f159678664b7b64d2ec2ae645a4500cc0
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
@@ -0,0 +1,927 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '3465_0'
+wandb_notes:
+  desc: null
+  value: 3465_0_restarted_1
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: false
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+restore_missing_checkpoint:
+  desc: null
+  value: null
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        crop_distractors_tlhw:
+        - 32
+        - 68
+        - 174
+        - 174
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: false
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+        crop_distractors_tlhw: null
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          crop_distractors_tlhw:
+          - 32
+          - 68
+          - 174
+          - 174
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+test:
+  desc: null
+  value:
+    username: leonmkim
+    hostname: grasp-login1
+    name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+    root_dir: ~/fish_leon
+    expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+    expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+    semantic_demo_grouping_name: semantic_demo_grouping.yaml
+    semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+    include_groups_list: all
+    batch_size: 128
+    store_dataset_in_memory: false
+    use_tb: true
+    local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup
+    local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1
+    resume_wandb_run: false
+feature_type:
+  desc: null
+  value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 20
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1737244125
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..dcf73113d3beba789cb0f6e3642a224a5aee2c09
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch
@@ -0,0 +1,89 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0b377ae..cda0fb8 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -132,8 +132,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # wandb_run_id: '1000_0' # all
+ 
+ # all books, 10/20 demos per book
++# crp_D+act history
++# wandb_run_id: '3456_0' 
++# wandb_run_id: '3456_1' # seed 1
++# wandb_run_id: '3456_2' # seed 2
++
++# all books, 06/20 demos per book
++# crp_D+act history
++wandb_run_id: '3465_0'
++# wandb_run_id: '3465_1' # seed 1
++# wandb_run_id: '3465_2' # seed 2
++
+ # RGBD+mask+act history
+-wandb_run_id: '1002_0' 
+ # wandb_run_id: '1009_1' # seed 1
+ # wandb_run_id: '1022_0' # dataset shuffle seed 1
+ 
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index e3c7dc1..19a2847 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -167,12 +167,18 @@ run_id_list = [
+     # '1948_0',
+     # '1950_0',
+     # '1954_1',
+-    '1002_0',
+-    '1009_1',
+-    '1022_0',
+-    '1003_0',
+-    '1013_1',
+-    '1017_0',
++    # '1002_0',
++    # '1009_1',
++    # '1022_0',
++    # '1003_0',
++    # '1013_1',
++    # '1017_0',
++    # '3456_0',
++    # '3456_1',
++    # '3456_2',
++    '3465_0',
++    '3465_1',
++    '3465_2',
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py
+index 0ad179a..f5cfc13 100644
+--- a/FISH/eval_policy.py
++++ b/FISH/eval_policy.py
+@@ -89,9 +89,10 @@ class Workspace:
+ 		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+ 		with open_dict(self.cfg):
+ 			self.cfg.feature_type = get_feature_dirname_from_configs(
+-				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+-				self.cfg.agent.config.policy_cfg.input_shapes,
+-				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
++				hydra.utils.instantiate(self.cfg.agent.config), 
++				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
++				# self.cfg.agent.config.policy_cfg.input_shapes,
++				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+ 			)
+ 
+ 		if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py
+index 50493fe..905b769 100644
+--- a/FISH/eval_robot.py
++++ b/FISH/eval_robot.py
+@@ -83,9 +83,10 @@ class Workspace:
+ 		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+ 		with open_dict(self.cfg):
+ 			self.cfg.feature_type = get_feature_dirname_from_configs(
+-				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+-				self.cfg.agent.config.policy_cfg.input_shapes,
+-				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
++				hydra.utils.instantiate(self.cfg.agent.config), 
++				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
++				# self.cfg.agent.config.policy_cfg.input_shapes,
++				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+ 			)
+ 
+ 		wandb_config = OmegaConf.to_container(
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..0a5e275d472d753e6ff46934c7da03511a1a0a47
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.65, 0.3739820122970847, 0.8550870119815186, 0.41187435390327565, 0.8174487351736318, 0.4792180508416731, 0.8044113814221957, 0.4990433230998045, 0.7912463805987083]]}
\ No newline at end of file
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..4138aae40c26cc6f2b4980263a49b28ff7985c83
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[13, 20, 0.65, 0.39654880764169254, 0.8603245168203681, 0.4385311752680692, 0.8164868056141166, 0.49042256176663257, 0.7910831644548946, 0.4824050068890475, 0.7813007085952826]]}
\ No newline at end of file
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4b88b028e3d9b6311b8b139bf84e652f83215879
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c4b549c276868919c0f491a0b92fc7d64d18307e5b08b0aa71ecf98ae9c0395
+size 1183093
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..447afece1c17d9221efe20dad08fa7761c93326b
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f1bedc7d64acfd933487bd4ba577ee603bd72ff2ee29025f80da331a2adcbec
+size 1161488
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1d0a6932e280c93bfee5a278405ea411128c4060
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:603d199abb1a8aa26ee32b8e11cded12fb329ffb576d7926db6d9b72da16d453
+size 1096372
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..49ba6454cb2834d67f8339a69622f581e94b63e0
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:815bf7acfa2781cd7af56580908e50b744e833fb44d89be5ad368026fd0ae034
+size 1093953
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6e5bad01996e0a43db8e991b1b4d4fde83b97635
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be54cbdf6a2c74d5872d0943711b9fe5c08511a91299f75ff586a67d760c839e
+size 1426112
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0ef4c71c1548ed7e577939201de0a93f24f6002d
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015a814176a13d9d5eb6c8534c22d3b4b7ece0cf09da24ca48842f587c3ce754
+size 1179201
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6cc0d5e45e2580bb9f0c7222c7e5a9c0f8a712e9
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fddbe9cae0cebc69c5e804deeddb6b643f5c55a27d5b00e57a3401fb2cc80b20
+size 1525087
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f4a388be2be4f1e2116f5a6965d569c41594e046
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b69f164b68c214c27944f0cc4822c6a56de1346eff9f1b034e2b58c05e0773e
+size 1453552
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f818515be95d24ce6761492f48d5d96007b5ef81
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97651cd5baa14e04e160fdb97e27e86d4c5d717eef4b18d53243b5d5e7de59e9
+size 1165331
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0eaef10c1000da3d0b99cbbba973d517afdf4179
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b8fe7251cc74d7e3aaab94983c785f28362553c3e41f0961ef1c5ab149d7bd1
+size 1457605
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..14d7f80e8fde7cc5b749ee303e9a9b7352318233
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c92e633fda071abe0bf478353c4246277cc742804c3416ec17efc035ad2372d
+size 1584151
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9ef044ff73a7312ba5c5720681f4a0cbec1b91a2
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c73c1e8035e9955d130d826b1f6ed12daebb02196b107dbcbd450af0ab8616
+size 1178950
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ada7cc3f6f4e61cfee2b9bb1da923fa2def01d40
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d91598e5051835f677e846bba3b3b20182672d199ba7b980b67bae56e49ef01d
+size 1178401
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..baaa17ea366c5a3ec61225c1cc6ec06d220b1936
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6706618b69c2f852f27029cd3f2623d90627580203ae03adeb4efded570bea41
+size 1165610
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d1661d164433d6537313e819ad1b4526177e2540
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae4f4aa5127f311e839c042eb90d298ecf8855e044de2406195f4cee84bcfbff
+size 1155941
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..254272d117860b6c81aefaa5b9da9b1c14f0e683
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15c189fb02c615acea539d0a97915ccf4d44cd14689c04e50ee7f901c5ea5f2d
+size 1176984
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4e50d0cf2fd2f15279ede6d1ddee6483c75b16cb
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cccf0749ac898e59a2f97b1f9f9697afc18dd70cb85e8188a0ac5dae96fc2b46
+size 1121447
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a9a81fccdb7c3ef2402aa50b76da8e0f5391c5be
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21e28c4cb1d06fc94de3ad9d2764e5465abc5297bca213abd8a93ff403f53385
+size 1091786
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..b739b4c405042cc13aa2287dcbc64b79171eebe8
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7ac84718a1584e3ff200aae1c4f78cb649f64896fcf503ac4175073d3e090ce
+size 1305681
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4 b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a5d045fcdc6d5b4027f2c76af5aa4c9c5b297ba5
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1569ddaa0ccfb738c03bdf185527443c7f965b1b7aefa2a7074d57b151b7771
+size 1347834
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..94b5615ab845d7ec638581162eb3e39c3c13eee7
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
@@ -0,0 +1,1121 @@
+
+loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1737244224.759597]: resetting environment
+[INFO] [1737244224.762974]: cleared current plan
+[INFO] [1737244224.764255]: moving to home
+[INFO] [1737244226.567127]: reached home
+[INFO] [1737244226.568414]: reset action history
+[INFO] [1737244229.279764]: environment reset
+Starting episode 0
+[INFO] [1737244229.284437]: resetting environment
+[INFO] [1737244229.288588]: cleared current plan
+[INFO] [1737244229.289377]: moving to home
+[INFO] [1737244230.291629]: reached home
+[INFO] [1737244230.293144]: reset action history
+[INFO] [1737244233.006393]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1737244235.143783]: resetting environment
+[INFO] [1737244235.146774]: cleared current plan
+[INFO] [1737244235.148013]: moving to home
+[INFO] [1737244236.149278]: reached home
+[INFO] [1737244236.150516]: reset action history
+[INFO] [1737244238.859939]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_0_2025-01-18-18-50-38.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:34521/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2553929]
+process[print_text-2]: started with pid [2553938]
+process[rosbag_record-3]: started with pid [2553953]
+started bagging!
+For topic gripper_width: timestamp difference is 100058748 for nearest: 1737244250468990280 - target: 1737244250368931532 at idx 160
+For topic gripper_width: timestamp difference is 101690565 for nearest: 1737244251268975054 - target: 1737244251167284489 at idx 169
+For topic gripper_width: timestamp difference is 103713100 for nearest: 1737244252268973653 - target: 1737244252165260553 at idx 182
+For topic gripper_width: timestamp difference is 105779368 for nearest: 1737244253269018132 - target: 1737244253163238764 at idx 192
+For topic gripper_width: timestamp difference is 106140347 for nearest: 1737244253468973608 - target: 1737244253362833261 at idx 194
+[INFO] [1737244265.858273]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1737244266.585463]: Stored episode 1.
+[INFO] [1737244266.585763]: Saving video...
+[INFO] [1737244266.795783]: Video saved!
+Starting episode 1
+[INFO] [1737244266.809893]: resetting environment
+[INFO] [1737244266.816082]: cleared current plan
+[INFO] [1737244266.816823]: moving to home
+[INFO] [1737244270.320009]: reached home
+[INFO] [1737244270.321003]: reset action history
+[INFO] [1737244273.029004]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244275.644117]: resetting environment
+[INFO] [1737244275.647629]: cleared current plan
+[INFO] [1737244275.648015]: moving to home
+[INFO] [1737244276.649272]: reached home
+[INFO] [1737244276.649709]: reset action history
+[INFO] [1737244279.357360]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_1_2025-01-18-18-51-19.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:42515/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2554070]
+process[print_text-5]: started with pid [2554071]
+process[rosbag_record-6]: started with pid [2554095]
+started bagging!
+For topic gripper_width: timestamp difference is -141652708 for nearest: 1737244284102305288 - target: 1737244284243957996 at idx 69
+[INFO] [1737244306.328598]: Storing episode...
+[rosbag_record-6] killing on exit[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1737244307.087942]: Stored episode 2.
+[INFO] [1737244307.088614]: Saving video...
+[INFO] [1737244307.771801]: Video saved!
+Starting episode 2
+[INFO] [1737244307.803307]: resetting environment
+[INFO] [1737244307.823995]: cleared current plan
+[INFO] [1737244307.824593]: moving to home
+[WARN] [1737244307.685409]: Plan exhausted
+[WARN] [1737244307.727655]: Plan exhausted
+[WARN] [1737244307.764996]: Plan exhausted
+[WARN] [1737244307.807154]: Plan exhausted
+[INFO] [1737244311.338103]: reached home
+[INFO] [1737244311.340324]: reset action history
+[INFO] [1737244314.110444]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244315.540929]: resetting environment
+[INFO] [1737244315.550048]: cleared current plan
+[INFO] [1737244315.553671]: moving to home
+[INFO] [1737244316.559528]: reached home
+[INFO] [1737244316.560876]: reset action history
+[INFO] [1737244319.274720]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_2_2025-01-18-18-51-59.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:42495/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2554227]
+process[print_text-8]: started with pid [2554228]
+process[rosbag_record-9]: started with pid [2554229]
+started bagging!
+For topic gripper_width: timestamp difference is -136138872 for nearest: 1737244342002312943 - target: 1737244342138451815 at idx 249
+[INFO] [1737244346.384523]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit
+[print_text-8] killing on exit
+[INFO] [1737244347.168899]: Stored episode 3.
+[INFO] [1737244347.175538]: Saving video...
+[INFO] [1737244347.858832]: Video saved!
+Starting episode 3
+[INFO] [1737244347.901076]: resetting environment
+[INFO] [1737244347.917847]: cleared current plan
+[INFO] [1737244347.918450]: moving to home
+[WARN] [1737244347.735132]: Plan exhausted
+[WARN] [1737244347.774282]: Plan exhausted
+[WARN] [1737244347.821746]: Plan exhausted
+[WARN] [1737244347.852708]: Plan exhausted
+[WARN] [1737244347.892819]: Plan exhausted
+[INFO] [1737244351.423486]: reached home
+[INFO] [1737244351.435666]: reset action history
+[INFO] [1737244354.169562]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244362.305911]: resetting environment
+[INFO] [1737244362.313941]: cleared current plan
+[INFO] [1737244362.315880]: moving to home
+[INFO] [1737244363.318597]: reached home
+[INFO] [1737244363.325046]: reset action history
+[INFO] [1737244366.049748]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_3_2025-01-18-18-52-46.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:45991/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2554338]
+process[print_text-11]: started with pid [2554339]
+process[rosbag_record-12]: started with pid [2554340]
+started bagging!
+[INFO] [1737244393.016217]: Storing episode...
+[print_text-11] killing on exit
+[rosbag_record-12] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1737244393.826168]: Stored episode 4.
+[INFO] [1737244393.830908]: Saving video...
+[WARN] [1737244394.172301]: Plan exhausted
+[WARN] [1737244394.213095]: Plan exhausted
+[WARN] [1737244394.264300]: Plan exhausted
+[WARN] [1737244394.292234]: Plan exhausted
+[WARN] [1737244394.333865]: Plan exhausted
+[WARN] [1737244394.372233]: Plan exhausted
+[WARN] [1737244394.412133]: Plan exhausted
+[WARN] [1737244394.452085]: Plan exhausted
+[WARN] [1737244394.492955]: Plan exhausted
+[INFO] [1737244394.441133]: Video saved!
+Starting episode 4
+[INFO] [1737244394.507562]: resetting environment
+[INFO] [1737244394.518236]: cleared current plan
+[INFO] [1737244394.521433]: moving to home
+[INFO] [1737244398.028082]: reached home
+[INFO] [1737244398.033086]: reset action history
+[INFO] [1737244400.866350]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244402.262473]: resetting environment
+[INFO] [1737244402.286627]: cleared current plan
+[INFO] [1737244402.306245]: moving to home
+[INFO] [1737244403.317129]: reached home
+[INFO] [1737244403.319258]: reset action history
+[INFO] [1737244406.096407]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_4_2025-01-18-18-53-26.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:44127/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2554494]
+process[print_text-14]: started with pid [2554495]
+process[rosbag_record-15]: started with pid [2554496]
+started bagging!
+For topic gripper_width: timestamp difference is -107391010 for nearest: 1737244418102304568 - target: 1737244418209695578 at idx 157
+[INFO] [1737244434.248903]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1737244435.044593]: Stored episode 5.
+[INFO] [1737244435.057912]: Saving video...
+[INFO] [1737244435.621897]: Video saved!
+Starting episode 5
+[INFO] [1737244435.701823]: resetting environment
+[INFO] [1737244435.714309]: cleared current plan
+[INFO] [1737244435.714949]: moving to home
+[WARN] [1737244435.415538]: Plan exhausted
+[WARN] [1737244435.455565]: Plan exhausted
+[WARN] [1737244435.501997]: Plan exhausted
+[WARN] [1737244435.534082]: Plan exhausted
+[WARN] [1737244435.573592]: Plan exhausted
+[WARN] [1737244435.612119]: Plan exhausted
+[WARN] [1737244435.654989]: Plan exhausted
+[WARN] [1737244435.695506]: Plan exhausted
+[INFO] [1737244439.220919]: reached home
+[INFO] [1737244439.229942]: reset action history
+[INFO] [1737244442.070722]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244445.500365]: resetting environment
+[INFO] [1737244445.526032]: cleared current plan
+[INFO] [1737244445.528103]: moving to home
+[INFO] [1737244446.536637]: reached home
+[INFO] [1737244446.543831]: reset action history
+[INFO] [1737244449.334843]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_5_2025-01-18-18-54-09.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:36703/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-16]: started with pid [2554625]
+process[print_text-17]: started with pid [2554626]
+process[rosbag_record-18]: started with pid [2554627]
+started bagging!
+For topic gripper_width: timestamp difference is 100535425 for nearest: 1737244455302336487 - target: 1737244455201801062 at idx 89
+[INFO] [1737244476.504533]: Storing episode...
+[rosbag_record-18] killing on exit
+[print_text-17] killing on exit
+[pub_text-16] killing on exit
+[INFO] [1737244477.306691]: Stored episode 6.
+[INFO] [1737244477.307303]: Saving video...
+[INFO] [1737244477.852011]: Video saved!
+Starting episode 6
+[INFO] [1737244477.932089]: resetting environment
+[INFO] [1737244477.954799]: cleared current plan
+[INFO] [1737244477.974901]: moving to home
+[WARN] [1737244477.856966]: Plan exhausted
+[WARN] [1737244477.899089]: Plan exhausted
+[WARN] [1737244477.934808]: Plan exhausted
+[INFO] [1737244481.182940]: reached home
+[INFO] [1737244481.193025]: reset action history
+[INFO] [1737244484.137588]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244489.698750]: resetting environment
+[INFO] [1737244489.707681]: cleared current plan
+[INFO] [1737244489.708019]: moving to home
+[INFO] [1737244490.710491]: reached home
+[INFO] [1737244490.715186]: reset action history
+[INFO] [1737244493.569893]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_6_2025-01-18-18-54-53.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:42537/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-19]: started with pid [2554758]
+process[print_text-20]: started with pid [2554759]
+process[rosbag_record-21]: started with pid [2554760]
+started bagging!
+For topic gripper_width: timestamp difference is 104729819 for nearest: 1737244516168971944 - target: 1737244516064242125 at idx 249
+[INFO] [1737244520.729051]: Storing episode...
+[rosbag_record-21] killing on exit
+[print_text-20] killing on exit
+[pub_text-19] killing on exit
+[INFO] [1737244521.540466]: Stored episode 7.
+[INFO] [1737244521.549278]: Saving video...
+[INFO] [1737244521.927653]: Video saved!
+Starting episode 7
+[INFO] [1737244522.003018]: resetting environment
+[INFO] [1737244522.032153]: cleared current plan
+[INFO] [1737244522.035942]: moving to home
+[WARN] [1737244521.894323]: Plan exhausted
+[WARN] [1737244521.933885]: Plan exhausted
+[WARN] [1737244521.972862]: Plan exhausted
+[WARN] [1737244522.012531]: Plan exhausted
+[INFO] [1737244525.553850]: reached home
+[INFO] [1737244525.575148]: reset action history
+[INFO] [1737244528.494942]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244530.266318]: resetting environment
+[INFO] [1737244530.298300]: cleared current plan
+[INFO] [1737244530.310706]: moving to home
+[INFO] [1737244531.331730]: reached home
+[INFO] [1737244531.335888]: reset action history
+[INFO] [1737244534.197067]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_7_2025-01-18-18-55-34.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:44471/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-22]: started with pid [2554867]
+process[print_text-23]: started with pid [2554868]
+process[rosbag_record-24]: started with pid [2554892]
+started bagging!
+For topic gripper_width: timestamp difference is 103897937 for nearest: 1737244540602264770 - target: 1737244540498366833 at idx 89
+For topic gripper_width: timestamp difference is 107821416 for nearest: 1737244547002261352 - target: 1737244546894439936 at idx 170
+For topic gripper_width: timestamp difference is -115312956 for nearest: 1737244559202304937 - target: 1737244559317617893 at idx 249
+[INFO] [1737244561.194358]: Storing episode...
+[rosbag_record-24] killing on exit
+[pub_text-22] killing on exit
+[print_text-23] killing on exit
+[INFO] [1737244562.010022]: Stored episode 8.
+[INFO] [1737244562.026453]: Saving video...
+[INFO] [1737244562.263587]: Video saved!
+Starting episode 8
+[INFO] [1737244562.377460]: resetting environment
+[INFO] [1737244562.415638]: cleared current plan
+[INFO] [1737244562.429173]: moving to home
+[INFO] [1737244565.847808]: reached home
+[INFO] [1737244565.853140]: reset action history
+[INFO] [1737244568.754037]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244571.571217]: resetting environment
+[INFO] [1737244571.602631]: cleared current plan
+[INFO] [1737244571.609715]: moving to home
+[INFO] [1737244572.629506]: reached home
+[INFO] [1737244572.633447]: reset action history
+[INFO] [1737244575.494560]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_8_2025-01-18-18-56-15.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:35577/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-25]: started with pid [2555025]
+process[print_text-26]: started with pid [2555026]
+process[rosbag_record-27]: started with pid [2555027]
+started bagging!
+For topic gripper_width: timestamp difference is 102120282 for nearest: 1737244580769003274 - target: 1737244580666882992 at idx 66
+For topic gripper_width: timestamp difference is 106760054 for nearest: 1737244586968971043 - target: 1737244586862210989 at idx 144
+For topic gripper_width: timestamp difference is 109444502 for nearest: 1737244590368973616 - target: 1737244590259529114 at idx 189
+For topic gripper_width: timestamp difference is 109642719 for nearest: 1737244590568997120 - target: 1737244590459354401 at idx 191
+For topic gripper_width: timestamp difference is 110280246 for nearest: 1737244591368970842 - target: 1737244591258690596 at idx 198
+For topic gripper_width: timestamp difference is 110956020 for nearest: 1737244592168973035 - target: 1737244592058017015 at idx 206
+[INFO] [1737244606.360599]: Storing episode...
+[pub_text-25] killing on exit
+[rosbag_record-27] killing on exit
+[print_text-26] killing on exit
+[INFO] [1737244607.147812]: Stored episode 9.
+[INFO] [1737244607.156749]: Saving video...
+[INFO] [1737244607.557261]: Video saved!
+Starting episode 9
+[INFO] [1737244607.669299]: resetting environment
+[INFO] [1737244607.686164]: cleared current plan
+[INFO] [1737244607.686431]: moving to home
+[INFO] [1737244611.089074]: reached home
+[INFO] [1737244611.110758]: reset action history
+[INFO] [1737244614.100710]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244621.572892]: resetting environment
+[INFO] [1737244621.601739]: cleared current plan
+[INFO] [1737244621.628117]: moving to home
+[INFO] [1737244622.634755]: reached home
+[INFO] [1737244622.661542]: reset action history
+[INFO] [1737244625.683270]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_9_2025-01-18-18-57-05.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:32949/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-28]: started with pid [2555156]
+process[print_text-29]: started with pid [2555157]
+process[rosbag_record-30]: started with pid [2555158]
+started bagging!
+For topic gripper_width: timestamp difference is 112329045 for nearest: 1737244634968968907 - target: 1737244634856639862 at idx 118
+[INFO] [1737244657.137026]: Storing episode...
+[print_text-29] killing on exit
+[rosbag_record-30] killing on exit
+[pub_text-28] killing on exit
+[INFO] [1737244657.981763]: Stored episode 10.
+[INFO] [1737244657.992483]: Saving video...
+[INFO] [1737244658.458525]: Video saved!
+Starting episode 10
+[INFO] [1737244658.597389]: resetting environment
+[INFO] [1737244658.612862]: cleared current plan
+[INFO] [1737244658.627331]: moving to home
+[WARN] [1737244658.499720]: Plan exhausted
+[WARN] [1737244658.538076]: Plan exhausted
+[WARN] [1737244658.572081]: Plan exhausted
+[INFO] [1737244662.234404]: reached home
+[INFO] [1737244662.245801]: reset action history
+[INFO] [1737244665.252826]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244668.551070]: resetting environment
+[INFO] [1737244668.624373]: cleared current plan
+[INFO] [1737244668.629595]: moving to home
+[INFO] [1737244669.653272]: reached home
+[INFO] [1737244669.666616]: reset action history
+[INFO] [1737244672.684605]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_10_2025-01-18-18-57-52.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:35009/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-31]: started with pid [2555290]
+process[print_text-32]: started with pid [2555291]
+process[rosbag_record-33]: started with pid [2555292]
+started bagging!
+For topic gripper_width: timestamp difference is 110265840 for nearest: 1737244673468970884 - target: 1737244673358705044 at idx 13
+For topic gripper_width: timestamp difference is -277414302 for nearest: 1737244694902305623 - target: 1737244695179719925 at idx 249
+For topic gripper_width: timestamp difference is -183037722 for nearest: 1737244696202337301 - target: 1737244696385375023 at idx 249
+[INFO] [1737244699.847916]: Storing episode...
+[pub_text-31] killing on exit
+[rosbag_record-33] killing on exit
+[print_text-32] killing on exit
+[INFO] [1737244700.676537]: Stored episode 11.
+[INFO] [1737244700.686037]: Saving video...
+[INFO] [1737244701.056384]: Video saved!
+Starting episode 11
+[INFO] [1737244701.157746]: resetting environment
+[INFO] [1737244701.185461]: cleared current plan
+[INFO] [1737244701.199672]: moving to home
+[INFO] [1737244704.833744]: reached home
+[INFO] [1737244704.849268]: reset action history
+[INFO] [1737244707.885051]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244711.178737]: resetting environment
+[INFO] [1737244711.242491]: cleared current plan
+[INFO] [1737244711.250300]: moving to home
+[INFO] [1737244712.260932]: reached home
+[INFO] [1737244712.272116]: reset action history
+[INFO] [1737244715.210774]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_11_2025-01-18-18-58-35.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:36411/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-34]: started with pid [2555422]
+process[print_text-35]: started with pid [2555423]
+process[rosbag_record-36]: started with pid [2555424]
+started bagging!
+For topic gripper_width: timestamp difference is 106265817 for nearest: 1737244715868994270 - target: 1737244715762728453 at idx 10
+For topic gripper_width: timestamp difference is 106654871 for nearest: 1737244716068999517 - target: 1737244715962344646 at idx 12
+For topic gripper_width: timestamp difference is 107064861 for nearest: 1737244716269027847 - target: 1737244716161962986 at idx 14
+For topic gripper_width: timestamp difference is 112610589 for nearest: 1737244719068993817 - target: 1737244718956383228 at idx 48
+For topic gripper_width: timestamp difference is 101430311 for nearest: 1737244739268969669 - target: 1737244739167539358 at idx 249
+For topic gripper_width: timestamp difference is 100378239 for nearest: 1737244739468916619 - target: 1737244739368538380 at idx 249
+[INFO] [1737244742.247252]: Storing episode...
+[print_text-35] killing on exit
+[rosbag_record-36] killing on exit
+[pub_text-34] killing on exit
+[WARN] [1737244743.573442]: Plan exhausted
+[INFO] [1737244743.068952]: Stored episode 12.
+[INFO] [1737244743.087804]: Saving video...
+[INFO] [1737244743.670010]: Video saved!
+Starting episode 12
+[INFO] [1737244743.839176]: resetting environment
+[INFO] [1737244743.868143]: cleared current plan
+[INFO] [1737244743.870674]: moving to home
+[WARN] [1737244743.616597]: Plan exhausted
+[WARN] [1737244743.654940]: Plan exhausted
+[WARN] [1737244743.692344]: Plan exhausted
+[WARN] [1737244743.732078]: Plan exhausted
+[WARN] [1737244743.772443]: Plan exhausted
+[WARN] [1737244743.820467]: Plan exhausted
+[WARN] [1737244743.852101]: Plan exhausted
+[INFO] [1737244747.381459]: reached home
+[INFO] [1737244747.390494]: reset action history
+[INFO] [1737244750.549637]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244752.810845]: resetting environment
+[INFO] [1737244752.885583]: cleared current plan
+[INFO] [1737244752.915950]: moving to home
+[INFO] [1737244753.942643]: reached home
+[INFO] [1737244753.964367]: reset action history
+[INFO] [1737244757.019066]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_12_2025-01-18-18-59-17.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:39041/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-37]: started with pid [2555556]
+process[print_text-38]: started with pid [2555557]
+process[rosbag_record-39]: started with pid [2555558]
+started bagging!
+For topic gripper_width: timestamp difference is -116103355 for nearest: 1737244764602304038 - target: 1737244764718407393 at idx 107
+[INFO] [1737244784.169092]: Storing episode...
+[rosbag_record-39] killing on exit
+[print_text-38] killing on exit
+[pub_text-37] killing on exit
+[WARN] [1737244785.499671]: Plan exhausted
+[WARN] [1737244785.541422]: Plan exhausted
+[WARN] [1737244785.622898]: Plan exhausted
+[WARN] [1737244785.643491]: Plan exhausted
+[WARN] [1737244785.652344]: Plan exhausted
+[INFO] [1737244785.012696]: Stored episode 13.
+[INFO] [1737244785.025650]: Saving video...
+[INFO] [1737244785.388976]: Video saved!
+Starting episode 13
+[INFO] [1737244785.600331]: resetting environment
+[INFO] [1737244785.659881]: cleared current plan
+[INFO] [1737244785.662280]: moving to home
+[INFO] [1737244789.074287]: reached home
+[INFO] [1737244789.089160]: reset action history
+[INFO] [1737244792.254403]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244795.789752]: resetting environment
+[INFO] [1737244795.854490]: cleared current plan
+[INFO] [1737244795.860146]: moving to home
+[INFO] [1737244796.887132]: reached home
+[INFO] [1737244796.903244]: reset action history
+[INFO] [1737244800.036996]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_13_2025-01-18-19-00-00.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:33867/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-40]: started with pid [2555689]
+process[print_text-41]: started with pid [2555690]
+process[rosbag_record-42]: started with pid [2555691]
+started bagging!
+[INFO] [1737244834.134137]: Storing episode...
+[pub_text-40] killing on exit[rosbag_record-42] killing on exit
+[print_text-41] killing on exit
+[WARN] [1737244835.505820]: Plan exhausted
+[WARN] [1737244835.540599]: Plan exhausted
+[WARN] [1737244835.572969]: Plan exhausted
+[WARN] [1737244835.613029]: Plan exhausted
+[WARN] [1737244835.654043]: Plan exhausted
+[WARN] [1737244835.694482]: Plan exhausted
+[INFO] [1737244835.018244]: Stored episode 14.
+[INFO] [1737244835.042652]: Saving video...
+[INFO] [1737244835.674878]: Video saved!
+Starting episode 14
+[INFO] [1737244835.810505]: resetting environment
+[INFO] [1737244835.837207]: cleared current plan
+[INFO] [1737244835.842030]: moving to home
+[WARN] [1737244835.732072]: Plan exhausted
+[WARN] [1737244835.772076]: Plan exhausted
+[WARN] [1737244835.816074]: Plan exhausted
+[INFO] [1737244839.063896]: reached home
+[INFO] [1737244839.074219]: reset action history
+[INFO] [1737244842.132271]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244849.723887]: resetting environment
+[INFO] [1737244849.774611]: cleared current plan
+[INFO] [1737244849.780562]: moving to home
+[INFO] [1737244850.809763]: reached home
+[INFO] [1737244850.835862]: reset action history
+[INFO] [1737244854.007453]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_14_2025-01-18-19-00-54.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:42883/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-43]: started with pid [2555799]
+process[print_text-44]: started with pid [2555823]
+process[rosbag_record-45]: started with pid [2555824]
+started bagging!
+For topic gripper_width: timestamp difference is -256719198 for nearest: 1737244879302302037 - target: 1737244879559021235 at idx 249
+[INFO] [1737244881.022517]: Storing episode...
+[print_text-44] killing on exit
+[pub_text-43] killing on exit[rosbag_record-45] killing on exit
+[INFO] [1737244881.897965]: Stored episode 15.
+[INFO] [1737244881.937660]: Saving video...
+[INFO] [1737244882.486806]: Video saved!
+Starting episode 15
+[INFO] [1737244882.695459]: resetting environment
+[INFO] [1737244882.800037]: cleared current plan
+[INFO] [1737244882.809729]: moving to home
+[WARN] [1737244882.372098]: Plan exhausted
+[WARN] [1737244882.412089]: Plan exhausted
+[WARN] [1737244882.453807]: Plan exhausted
+[WARN] [1737244882.506328]: Plan exhausted
+[WARN] [1737244882.541392]: Plan exhausted
+[WARN] [1737244882.593117]: Plan exhausted
+[WARN] [1737244882.625292]: Plan exhausted
+[WARN] [1737244882.681688]: Plan exhausted
+[WARN] [1737244882.709916]: Plan exhausted
+[WARN] [1737244882.739711]: Plan exhausted
+[WARN] [1737244882.773750]: Plan exhausted
+[INFO] [1737244886.160906]: reached home
+[INFO] [1737244886.172546]: reset action history
+[INFO] [1737244889.400177]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244889.819566]: resetting environment
+[INFO] [1737244889.823210]: cleared current plan
+[INFO] [1737244889.823405]: moving to home
+[INFO] [1737244890.831231]: reached home
+[INFO] [1737244890.843810]: reset action history
+[INFO] [1737244894.063153]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_15_2025-01-18-19-01-34.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:41397/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-46]: started with pid [2555954]
+process[print_text-47]: started with pid [2555955]
+process[rosbag_record-48]: started with pid [2555956]
+started bagging!
+[INFO] [1737244930.184277]: Storing episode...
+[rosbag_record-48] killing on exit
+[print_text-47] killing on exit
+[pub_text-46] killing on exit
+[WARN] [1737244931.532806]: Plan exhausted
+[WARN] [1737244931.577328]: Plan exhausted
+[WARN] [1737244931.618598]: Plan exhausted
+[WARN] [1737244931.652093]: Plan exhausted
+[WARN] [1737244931.702081]: Plan exhausted
+[WARN] [1737244931.740587]: Plan exhausted
+[WARN] [1737244931.777574]: Plan exhausted
+[INFO] [1737244931.053045]: Stored episode 16.
+[INFO] [1737244931.065941]: Saving video...
+[INFO] [1737244931.803931]: Video saved!
+Starting episode 16
+[INFO] [1737244931.957258]: resetting environment
+[INFO] [1737244931.983476]: cleared current plan
+[INFO] [1737244931.986443]: moving to home
+[WARN] [1737244931.813815]: Plan exhausted
+[WARN] [1737244931.855700]: Plan exhausted
+[WARN] [1737244931.895912]: Plan exhausted
+[WARN] [1737244931.935800]: Plan exhausted
+[WARN] [1737244931.972525]: Plan exhausted
+[INFO] [1737244935.497221]: reached home
+[INFO] [1737244935.567304]: reset action history
+proceeding to start episode!
+[INFO] [1737244938.767103]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737244970.217188]: resetting environment
+[INFO] [1737244970.278834]: cleared current plan
+[INFO] [1737244970.285641]: moving to home
+[INFO] [1737244971.298534]: reached home
+[INFO] [1737244971.315233]: reset action history
+[INFO] [1737244974.446896]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_16_2025-01-18-19-02-54.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:46495/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-49]: started with pid [2556100]
+process[print_text-50]: started with pid [2556101]
+process[rosbag_record-51]: started with pid [2556102]
+started bagging!
+For topic gripper_width: timestamp difference is 104171520 for nearest: 1737244993002299791 - target: 1737244992898128271 at idx 249
+For topic gripper_width: timestamp difference is -100088058 for nearest: 1737245005202301564 - target: 1737245005302389622 at idx 249
+[INFO] [1737245010.938462]: Storing episode...
+[pub_text-49] killing on exit
+[rosbag_record-51] killing on exit
+[print_text-50] killing on exit
+[INFO] [1737245011.817703]: Stored episode 17.
+[INFO] [1737245011.848894]: Saving video...
+[INFO] [1737245012.435195]: Video saved!
+Starting episode 17
+[INFO] [1737245012.567984]: resetting environment
+[INFO] [1737245012.610405]: cleared current plan
+[INFO] [1737245012.612073]: moving to home
+[WARN] [1737245012.095436]: Plan exhausted
+[WARN] [1737245012.132090]: Plan exhausted
+[WARN] [1737245012.172445]: Plan exhausted
+[WARN] [1737245012.213290]: Plan exhausted
+[WARN] [1737245012.252352]: Plan exhausted
+[WARN] [1737245012.292124]: Plan exhausted
+[WARN] [1737245012.335131]: Plan exhausted
+[WARN] [1737245012.380080]: Plan exhausted
+[WARN] [1737245012.412090]: Plan exhausted
+[WARN] [1737245012.452565]: Plan exhausted
+[WARN] [1737245012.492255]: Plan exhausted
+[WARN] [1737245012.534642]: Plan exhausted
+[WARN] [1737245012.576740]: Plan exhausted
+[INFO] [1737245016.136094]: reached home
+[INFO] [1737245016.183084]: reset action history
+[INFO] [1737245019.312787]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737245047.999920]: resetting environment
+[INFO] [1737245048.046611]: cleared current plan
+[INFO] [1737245048.060864]: moving to home
+[INFO] [1737245049.069917]: reached home
+[INFO] [1737245049.103320]: reset action history
+[INFO] [1737245052.190992]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_17_2025-01-18-19-04-12.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:38849/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-52]: started with pid [2556811]
+process[print_text-53]: started with pid [2556812]
+process[rosbag_record-54]: started with pid [2556813]
+started bagging!
+For topic gripper_width: timestamp difference is -143369209 for nearest: 1737245061002334345 - target: 1737245061145703554 at idx 116
+For topic gripper_width: timestamp difference is -140898197 for nearest: 1737245064202264578 - target: 1737245064343162775 at idx 161
+[INFO] [1737245079.428406]: Storing episode...
+[pub_text-52] killing on exit
+[rosbag_record-54] killing on exit[print_text-53] killing on exit
+[INFO] [1737245080.255833]: Stored episode 18.
+[INFO] [1737245080.272668]: Saving video...
+[INFO] [1737245080.814178]: Video saved!
+Starting episode 18
+[INFO] [1737245081.018546]: resetting environment
+[INFO] [1737245081.065175]: cleared current plan
+[INFO] [1737245081.065392]: moving to home
+[WARN] [1737245080.775527]: Plan exhausted
+[WARN] [1737245080.827695]: Plan exhausted
+[WARN] [1737245080.876751]: Plan exhausted
+[WARN] [1737245080.893277]: Plan exhausted
+[WARN] [1737245080.943944]: Plan exhausted
+[WARN] [1737245080.977526]: Plan exhausted
+[WARN] [1737245081.028535]: Plan exhausted
+[WARN] [1737245081.053602]: Plan exhausted
+[INFO] [1737245084.678144]: reached home
+[INFO] [1737245084.735219]: reset action history
+[INFO] [1737245088.004074]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737245098.924999]: resetting environment
+[INFO] [1737245099.040285]: cleared current plan
+[INFO] [1737245099.079151]: moving to home
+[INFO] [1737245100.117721]: reached home
+[INFO] [1737245100.139270]: reset action history
+[INFO] [1737245103.454646]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_18_2025-01-18-19-05-03.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:36699/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-55]: started with pid [2556955]
+process[print_text-56]: started with pid [2556956]
+process[rosbag_record-57]: started with pid [2556957]
+started bagging!
+For topic gripper_width: timestamp difference is 100194951 for nearest: 1737245114968967219 - target: 1737245114868772268 at idx 157
+[INFO] [1737245139.175689]: Storing episode...
+[print_text-56] killing on exit
+[rosbag_record-57] killing on exit
+[pub_text-55] killing on exit
+[INFO] [1737245140.058864]: Stored episode 19.
+[INFO] [1737245140.070797]: Saving video...
+[INFO] [1737245140.508326]: Video saved!
+Starting episode 19
+[INFO] [1737245140.697055]: resetting environment
+[INFO] [1737245140.741572]: cleared current plan
+[INFO] [1737245140.741955]: moving to home
+[WARN] [1737245140.492090]: Plan exhausted
+[WARN] [1737245140.534747]: Plan exhausted
+[WARN] [1737245140.605132]: Plan exhausted
+[WARN] [1737245140.622823]: Plan exhausted
+[WARN] [1737245140.653386]: Plan exhausted
+[WARN] [1737245140.707491]: Plan exhausted
+[WARN] [1737245140.733832]: Plan exhausted
+[INFO] [1737245144.251467]: reached home
+[INFO] [1737245144.268507]: reset action history
+[INFO] [1737245147.550928]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737245150.414596]: resetting environment
+[INFO] [1737245150.450903]: cleared current plan
+[INFO] [1737245150.454060]: moving to home
+[INFO] [1737245151.458857]: reached home
+[INFO] [1737245151.460509]: reset action history
+[INFO] [1737245154.864884]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/episode_rosbags/episode_19_2025-01-18-19-05-54.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2553479.log
+started roslaunch server http://158.130.50.37:35419/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-58]: started with pid [2557099]
+process[print_text-59]: started with pid [2557100]
+process[rosbag_record-60]: started with pid [2557101]
+started bagging!
+For topic gripper_width: timestamp difference is -107046490 for nearest: 1737245188502300377 - target: 1737245188609346867 at idx 249
+[INFO] [1737245193.256547]: Storing episode...
+[pub_text-58] killing on exit
+[rosbag_record-60] killing on exit
+[print_text-59] killing on exit
+[INFO] [1737245194.119194]: Stored episode 20.
+[INFO] [1737245194.125457]: Saving video...
+[INFO] [1737245194.505436]: Video saved!
+[INFO] [1737245194.682159]: resetting environment
+[INFO] [1737245194.747294]: cleared current plan
+[INFO] [1737245194.749739]: moving to home
+[WARN] [1737245194.412101]: Plan exhausted
+[WARN] [1737245194.455735]: Plan exhausted
+[WARN] [1737245194.493567]: Plan exhausted
+[WARN] [1737245194.532877]: Plan exhausted
+[WARN] [1737245194.591003]: Plan exhausted
+[WARN] [1737245194.614417]: Plan exhausted
+[WARN] [1737245194.656694]: Plan exhausted
+[WARN] [1737245194.696195]: Plan exhausted
+[WARN] [1737245194.740471]: Plan exhausted
+[INFO] [1737245198.056172]: reached home
+[INFO] [1737245198.060249]: reset action history
+[INFO] [1737245201.290338]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
new file mode 100644
index 0000000000000000000000000000000000000000..088fb15d1989884bd7bc0f375829847c404e20e0
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
@@ -0,0 +1,106 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index a5e82a4..cda0fb8 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17
+ 
+ # Number of evaluation trajectories
+ # num_eval: 10
+-num_eval: 5
++num_eval: 20
+ 
+ debug_timestamps: False
+ open_loop: False
+@@ -132,15 +132,25 @@ load_checkpoint: ${agent.load_checkpoint}
+ # wandb_run_id: '1000_0' # all
+ 
+ # all books, 10/20 demos per book
++# crp_D+act history
++# wandb_run_id: '3456_0' 
++# wandb_run_id: '3456_1' # seed 1
++# wandb_run_id: '3456_2' # seed 2
++
++# all books, 06/20 demos per book
++# crp_D+act history
++wandb_run_id: '3465_0'
++# wandb_run_id: '3465_1' # seed 1
++# wandb_run_id: '3465_2' # seed 2
++
+ # RGBD+mask+act history
+-# wandb_run_id: '1002_0' 
+ # wandb_run_id: '1009_1' # seed 1
+ # wandb_run_id: '1022_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1003_0'
+ # wandb_run_id: '1013_1' # seed 1
+-wandb_run_id: '1017_0' # dataset shuffle seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index e3c7dc1..19a2847 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -167,12 +167,18 @@ run_id_list = [
+     # '1948_0',
+     # '1950_0',
+     # '1954_1',
+-    '1002_0',
+-    '1009_1',
+-    '1022_0',
+-    '1003_0',
+-    '1013_1',
+-    '1017_0',
++    # '1002_0',
++    # '1009_1',
++    # '1022_0',
++    # '1003_0',
++    # '1013_1',
++    # '1017_0',
++    # '3456_0',
++    # '3456_1',
++    # '3456_2',
++    '3465_0',
++    '3465_1',
++    '3465_2',
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py
+index 0ad179a..f5cfc13 100644
+--- a/FISH/eval_policy.py
++++ b/FISH/eval_policy.py
+@@ -89,9 +89,10 @@ class Workspace:
+ 		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+ 		with open_dict(self.cfg):
+ 			self.cfg.feature_type = get_feature_dirname_from_configs(
+-				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+-				self.cfg.agent.config.policy_cfg.input_shapes,
+-				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
++				hydra.utils.instantiate(self.cfg.agent.config), 
++				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
++				# self.cfg.agent.config.policy_cfg.input_shapes,
++				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+ 			)
+ 
+ 		if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py
+index 50493fe..905b769 100644
+--- a/FISH/eval_robot.py
++++ b/FISH/eval_robot.py
+@@ -83,9 +83,10 @@ class Workspace:
+ 		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+ 		with open_dict(self.cfg):
+ 			self.cfg.feature_type = get_feature_dirname_from_configs(
+-				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+-				self.cfg.agent.config.policy_cfg.input_shapes,
+-				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
++				hydra.utils.instantiate(self.cfg.agent.config), 
++				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
++				# self.cfg.agent.config.policy_cfg.input_shapes,
++				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+ 			)
+ 
+ 		wandb_config = OmegaConf.to_container(
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..c08de8809919811bf123824e636fbe0306759406
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-18T23:48:46.112659",
+    "startedAt": "2025-01-18T23:48:45.733511",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3752.0185,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.938,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4548.31,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 498.2482566833496
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a356e61a3732ce0c747577b5bc30802f511b3f7
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "9c4b549c276868919c0f491a0b92fc7d64d18307e5b08b0aa71ecf98ae9c0395", "size": 1183093, "path": "media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4"}, "global_step": 2655, "_timestamp": 1737245205.8003461, "_runtime": 1080.0591061115265, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "66c73c1e8035e9955d130d826b1f6ed12daebb02196b107dbcbd450af0ab8616", "size": 1178950, "path": "media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4"}, "eval/num_success": 13.0, "episode": 20.0, "eval/success_rate": 0.6499999761581421, "eval/success_rate_lb_0.01": 0.3739820122718811, "eval/success_rate_ub_0.01": 0.8550869822502136, "eval/success_rate_lb_0.025": 0.41187435388565063, "eval/success_rate_ub_0.025": 0.8174487352371216, "eval/success_rate_lb_0.05": 0.479218065738678, "eval/success_rate_ub_0.05": 0.8044113516807556, "eval/success_rate_lb_0.1": 0.4990433156490326, "eval/success_rate_ub_0.1": 0.7912463545799255, "eval/2_eval": {"_type": "video-file", "sha256": "d91598e5051835f677e846bba3b3b20182672d199ba7b980b67bae56e49ef01d", "size": 1178401, "path": "media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "6706618b69c2f852f27029cd3f2623d90627580203ae03adeb4efded570bea41", "size": 1165610, "path": "media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ae4f4aa5127f311e839c042eb90d298ecf8855e044de2406195f4cee84bcfbff", "size": 1155941, "path": "media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "15c189fb02c615acea539d0a97915ccf4d44cd14689c04e50ee7f901c5ea5f2d", "size": 1176984, "path": "media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "cccf0749ac898e59a2f97b1f9f9697afc18dd70cb85e8188a0ac5dae96fc2b46", "size": 1121447, "path": "media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "21e28c4cb1d06fc94de3ad9d2764e5465abc5297bca213abd8a93ff403f53385", "size": 1091786, "path": "media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "c7ac84718a1584e3ff200aae1c4f78cb649f64896fcf503ac4175073d3e090ce", "size": 1305681, "path": "media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "a1569ddaa0ccfb738c03bdf185527443c7f965b1b7aefa2a7074d57b151b7771", "size": 1347834, "path": "media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "9f1bedc7d64acfd933487bd4ba577ee603bd72ff2ee29025f80da331a2adcbec", "size": 1161488, "path": "media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "603d199abb1a8aa26ee32b8e11cded12fb329ffb576d7926db6d9b72da16d453", "size": 1096372, "path": "media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "815bf7acfa2781cd7af56580908e50b744e833fb44d89be5ad368026fd0ae034", "size": 1093953, "path": "media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "be54cbdf6a2c74d5872d0943711b9fe5c08511a91299f75ff586a67d760c839e", "size": 1426112, "path": "media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "015a814176a13d9d5eb6c8534c22d3b4b7ece0cf09da24ca48842f587c3ce754", "size": 1179201, "path": "media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "fddbe9cae0cebc69c5e804deeddb6b643f5c55a27d5b00e57a3401fb2cc80b20", "size": 1525087, "path": "media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "9b69f164b68c214c27944f0cc4822c6a56de1346eff9f1b034e2b58c05e0773e", "size": 1453552, "path": "media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "97651cd5baa14e04e160fdb97e27e86d4c5d717eef4b18d53243b5d5e7de59e9", "size": 1165331, "path": "media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "7b8fe7251cc74d7e3aaab94983c785f28362553c3e41f0961ef1c5ab149d7bd1", "size": 1457605, "path": "media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "2c92e633fda071abe0bf478353c4246277cc742804c3416ec17efc035ad2372d", "size": 1584151, "path": "media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "5376ac71dfa7920aa911ca52a8f1e0d53fb5cd04b130809bf65856f5905bffb4", "size": 400, "artifact_path": "wandb-client-artifact://hz9woewoywflhrw4k96sku5k4p8ugbrzxqht0f1uxfc5p2t8i5ydbfmrhqakz41sa87p125u1yhpi6n9mq3lnblbag19q83nupcxx52pyqbadjhjfhsghvxk30ixo1ov/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://kxgw5bjnyp21k0f3vdzom901xsvubballucquezc7kofpzjv91rtd8bs19v3wxnwtgcbb3kav1sosntojgqsv3b7kmaik6kty5u6zipx093a5u5mg73igt6vk5ukqall:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 13, "total_num_episodes": 20, "total_success_rate": 0.65, "total_success_rate_lb_0.01": 0.39654880764169254, "total_success_rate_ub_0.01": 0.8603245168203681, "total_success_rate_lb_0.025": 0.4385311752680692, "total_success_rate_ub_0.025": 0.8164868056141166, "total_success_rate_lb_0.05": 0.49042256176663257, "total_success_rate_ub_0.05": 0.7910831644548946, "total_success_rate_lb_0.1": 0.4824050068890475, "total_success_rate_ub_0.1": 0.7813007085952826, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "9ef68d5d87d1210dec34161c6cf3998d22b6e988329b167cd729c4c24839f71b", "size": 508, "artifact_path": "wandb-client-artifact://byapohygapok6fyyb6fr3yqdz0dje6ui4fksdhupbtrntdfpdxdmnqlq0m9iin840obd1j4ba58pb9zpi7sx9gxkn57fw2upfb3nr3jle2ya041p9qejjflnzvivgdar/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://tm4e5kzgshem3643y7qg6i63rl8olcos8has58pjmqnwsw973i9h63kdq99g4pmfb69lbgpeg1l2h44afrtysipy59pck837zg972u05jwwna53omvirfhn7rqfzavoq:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1093}, "eval/total_success_rate": 0.6499999761581421, "eval/total_success_rate_lb_0.01": 0.3965488076210022, "eval/total_success_rate_ub_0.01": 0.860324501991272, "eval/total_success_rate_lb_0.025": 0.43853119015693665, "eval/total_success_rate_ub_0.025": 0.8164867758750916, "eval/total_success_rate_lb_0.05": 0.4904225766658783, "eval/total_success_rate_ub_0.05": 0.7910831570625305, "eval/total_success_rate_lb_0.1": 0.48240500688552856, "eval/total_success_rate_ub_0.1": 0.7813007235527039}
\ No newline at end of file
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug-internal.log b/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..d7a2edf2da28ae0bc73b40228f8e143a559d5541
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug-internal.log
@@ -0,0 +1,2295 @@
+2025-01-18 18:48:45,741 INFO    StreamThr :2553803 [internal.py:wandb_internal():85] W&B internal server running at pid: 2553803, started at: 2025-01-18 18:48:45.740809
+2025-01-18 18:48:45,742 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status
+2025-01-18 18:48:45,754 INFO    WriterThread:2553803 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb
+2025-01-18 18:48:45,755 DEBUG   SenderThread:2553803 [sender.py:send():379] send: header
+2025-01-18 18:48:45,764 DEBUG   SenderThread:2553803 [sender.py:send():379] send: run
+2025-01-18 18:48:46,002 INFO    SenderThread:2553803 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files
+2025-01-18 18:48:46,002 INFO    SenderThread:2553803 [sender.py:_start_run_threads():1188] run started: q1ti958r with start time 1737244125.74124
+2025-01-18 18:48:46,019 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: check_version
+2025-01-18 18:48:46,019 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: check_version
+2025-01-18 18:48:46,095 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: run_start
+2025-01-18 18:48:46,099 DEBUG   HandlerThread:2553803 [system_info.py:__init__():26] System info init
+2025-01-18 18:48:46,099 DEBUG   HandlerThread:2553803 [system_info.py:__init__():41] System info init done
+2025-01-18 18:48:46,100 INFO    HandlerThread:2553803 [system_monitor.py:start():194] Starting system monitor
+2025-01-18 18:48:46,100 INFO    SystemMonitor:2553803 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-18 18:48:46,100 INFO    HandlerThread:2553803 [system_monitor.py:probe():214] Collecting system info
+2025-01-18 18:48:46,100 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started cpu monitoring
+2025-01-18 18:48:46,100 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started disk monitoring
+2025-01-18 18:48:46,102 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started gpu monitoring
+2025-01-18 18:48:46,102 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started memory monitoring
+2025-01-18 18:48:46,102 INFO    SystemMonitor:2553803 [interfaces.py:start():188] Started network monitoring
+2025-01-18 18:48:46,112 DEBUG   HandlerThread:2553803 [system_info.py:probe():152] Probing system
+2025-01-18 18:48:46,114 DEBUG   HandlerThread:2553803 [system_info.py:_probe_git():137] Probing git
+2025-01-18 18:48:46,119 DEBUG   HandlerThread:2553803 [system_info.py:_probe_git():145] Probing git done
+2025-01-18 18:48:46,119 DEBUG   HandlerThread:2553803 [system_info.py:probe():200] Probing system done
+2025-01-18 18:48:46,119 DEBUG   HandlerThread:2553803 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-18T23:48:46.112659', 'startedAt': '2025-01-18T23:48:45.733511', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3752.0185, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.938, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4548.31, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 498.2482566833496}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-18 18:48:46,120 INFO    HandlerThread:2553803 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-18 18:48:46,120 INFO    HandlerThread:2553803 [system_monitor.py:probe():227] Publishing system info
+2025-01-18 18:48:46,120 DEBUG   HandlerThread:2553803 [system_info.py:_save_code():44] Saving code
+2025-01-18 18:48:46,121 DEBUG   HandlerThread:2553803 [system_info.py:_save_code():67] Saving code done
+2025-01-18 18:48:46,121 DEBUG   HandlerThread:2553803 [system_info.py:_save_patches():84] Saving git patches
+2025-01-18 18:48:46,218 DEBUG   HandlerThread:2553803 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-18 18:48:46,219 INFO    HandlerThread:2553803 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-18 18:48:46,226 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:48:46,226 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-18 18:48:46,226 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-18 18:48:46,226 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-18 18:48:46,227 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now
+2025-01-18 18:48:46,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-18 18:48:46,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: python_packages
+2025-01-18 18:48:46,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:48:46,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:48:46,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:46,475 INFO    HandlerThread:2553803 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/tb"
+  save: true
+}
+control {
+  relay_id: "3ad0bdfad9e7418390329a14bb5b86ea"
+}
+_info {
+  stream_id: "q1ti958r"
+}
+
+2025-01-18 18:48:46,475 DEBUG   HandlerThread:2553803 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-18 18:48:46,492 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/7eqh4ohy-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
+2025-01-18 18:48:46,492 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/4vr9lobk-code/FISH/eval_robot.py
+2025-01-18 18:48:46,497 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/o1efaon9-wandb-metadata.json
+2025-01-18 18:48:46,522 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/zsz3gief-diff.patch
+2025-01-18 18:48:46,559 DEBUG   SenderThread:2553803 [sender.py:send():379] send: telemetry
+2025-01-18 18:48:46,560 DEBUG   SenderThread:2553803 [sender.py:send():379] send: tbrecord
+2025-01-18 18:48:46,560 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:48:46,560 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0 with policy live
+2025-01-18 18:48:46,744 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/k6c5de3k-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch
+2025-01-18 18:48:47,004 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code
+2025-01-18 18:48:47,005 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH
+2025-01-18 18:48:47,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:48,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:49,003 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:48:49,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:50,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:51,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:48:51,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:52,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:53,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:54,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:55,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:56,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:48:56,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:57,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:58,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:48:59,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:00,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:01,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:01,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:01,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:01,564 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:02,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:03,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:04,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:05,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:06,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:07,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:07,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:08,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:09,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:10,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:11,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:12,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:12,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:13,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:14,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:15,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:16,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:16,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:16,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:17,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:17,625 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:18,011 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 18:49:18,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:19,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:20,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:21,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:22,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:23,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:23,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:24,477 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:25,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:26,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:27,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:28,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:29,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:29,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:30,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:31,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:31,471 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:32,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:33,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:34,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:34,557 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:35,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:36,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:37,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:38,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:39,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:40,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:40,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:41,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:42,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:43,481 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:44,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:45,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:45,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:46,103 DEBUG   SystemMonitor:2553803 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-18 18:49:46,103 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:49:46,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:49:46,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:49:46,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:47,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:48,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:49,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:50,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:50,549 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:51,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:52,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:53,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:54,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:55,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:56,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:49:56,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:57,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:58,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:49:59,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:00,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:01,471 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:01,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:01,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:01,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:02,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:03,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:04,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:05,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:06,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:07,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:07,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:08,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:09,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:10,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:11,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:12,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:13,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:13,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:14,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:15,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:16,104 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:50:16,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:16,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:16,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:17,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:18,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:19,013 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:19,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:20,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:21,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:22,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:23,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:24,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:24,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:24,759 DEBUG   SenderThread:2553803 [sender.py:send():379] send: config
+2025-01-18 18:50:25,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:26,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:27,031 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:27,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:28,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:29,032 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:29,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:30,292 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:30,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:31,033 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:31,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:31,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:32,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:33,033 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:33,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:34,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:35,034 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:35,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:36,149 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:36,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:37,035 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:37,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:38,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:39,035 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:39,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:40,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:41,036 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:41,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:41,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:42,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:43,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:44,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:45,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:46,104 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:50:46,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:50:46,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:50:46,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:46,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:47,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:48,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:49,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:50,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:51,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:52,277 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:52,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:53,039 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:53,492 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:54,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:55,040 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:50:55,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:56,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:57,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:50:57,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:58,041 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 18:50:58,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:50:59,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:00,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:01,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:01,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:01,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:02,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:03,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:03,554 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:04,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:05,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:06,494 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:06,808 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:51:06,809 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4 with policy now
+2025-01-18 18:51:06,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:51:06,810 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:51:06,810 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:51:06,810 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:51:07,043 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos
+2025-01-18 18:51:07,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:51:07,074 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/t587w10y-media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 18:51:07,386 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/neoboxbz-media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 18:51:07,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:08,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:08,817 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:09,044 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:09,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:10,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:11,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:12,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:13,045 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:13,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:14,073 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:14,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:15,046 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:15,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:16,047 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:51:16,111 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:51:16,305 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fbld0vc8-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:51:16,472 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:16,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:16,496 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:17,047 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:17,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:18,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:19,048 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:19,357 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:19,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:20,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:21,049 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:21,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:22,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:23,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:24,377 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:24,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:25,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:26,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:27,051 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:27,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:28,498 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:29,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:29,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:30,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:31,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:31,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:32,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:33,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:34,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:34,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:35,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:36,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:37,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:38,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:39,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:40,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:40,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:41,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:42,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:43,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:44,500 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:45,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:46,105 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:51:46,106 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:46,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:51:46,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:51:46,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:47,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:47,799 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:51:47,799 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4 with policy now
+2025-01-18 18:51:47,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:51:47,800 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:51:47,800 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:51:47,801 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:51:48,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:51:48,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 18:51:48,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:51:48,059 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/u93kbo3q-media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 18:51:48,338 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/t0g3xcz1-media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 18:51:48,503 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:49,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:49,504 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:50,058 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:50,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:51,338 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:51,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:52,510 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:53,059 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:53,509 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:54,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:55,060 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:55,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:55,990 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:51:55,990 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:51:55,991 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:51:56,061 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:51:56,061 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:51:56,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:51:56,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:57,061 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:57,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:58,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:51:59,062 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:51:59,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:00,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:01,063 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:01,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:01,478 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:01,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:01,615 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:02,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:03,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:04,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:05,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:06,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:07,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:07,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:08,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:09,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:10,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:11,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:12,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:12,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:13,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:14,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:15,573 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:16,106 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:52:16,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:16,473 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:16,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:17,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:17,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:18,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:19,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:20,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:21,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:22,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:23,070 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:23,257 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:23,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:24,575 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:25,574 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:26,575 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:27,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:27,896 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:52:27,896 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4 with policy now
+2025-01-18 18:52:27,899 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:52:27,899 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:52:27,899 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:52:27,900 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:52:28,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:52:28,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 18:52:28,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:52:28,149 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/e8z32mvu-media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 18:52:28,431 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/6l74dbyw-media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 18:52:28,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:28,918 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:29,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:29,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:30,072 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:30,586 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:31,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:31,477 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:31,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:32,581 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:33,073 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:33,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:34,172 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:34,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:35,074 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:35,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:36,582 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:37,584 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:38,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:39,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:39,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:40,584 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:41,598 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:42,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:42,824 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:52:42,825 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:52:42,825 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:52:43,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:43,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:52:43,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:52:43,338 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ey0x170e-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:52:43,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:44,591 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:45,077 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:45,361 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:45,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:46,106 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:52:46,474 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:52:46,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:52:46,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:47,078 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:52:47,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:48,586 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:49,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:50,534 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:50,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:51,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:52,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:53,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:54,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:55,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:56,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:52:56,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:57,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:58,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:52:59,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:00,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:01,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:01,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:01,553 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:01,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:02,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:03,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:04,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:05,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:06,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:07,482 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:07,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:08,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:09,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:10,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:11,589 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:12,483 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:12,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:13,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:14,497 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:53:14,497 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4 with policy now
+2025-01-18 18:53:14,503 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:53:14,504 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:53:14,504 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:53:14,505 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:53:14,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:14,780 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/aqhxa1wy-media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 18:53:15,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:53:15,391 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/2467bf5q-media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 18:53:15,599 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:16,088 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:16,107 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:53:16,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:16,480 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:16,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:17,089 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:17,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:17,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:18,601 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:19,089 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:19,602 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:20,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:21,604 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:22,613 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:22,875 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:53:22,875 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:53:22,875 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:22,876 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:53:23,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:23,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:53:23,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:53:23,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:24,605 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:25,091 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:25,611 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:26,616 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:27,092 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:27,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:28,556 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:28,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:29,093 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:29,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:30,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:31,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:31,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:31,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:32,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:33,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:33,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:34,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:35,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:36,606 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:37,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:38,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:39,096 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:39,337 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:39,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:40,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:41,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:42,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:43,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:44,486 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:44,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:45,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:46,107 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:53:46,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:53:46,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:53:46,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:47,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:48,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:49,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:49,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:50,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:51,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:52,609 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:53,610 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:54,617 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:55,050 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:53:55,102 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:55,625 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:55,697 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:53:55,697 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:53:55,698 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4 with policy now
+2025-01-18 18:53:55,698 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:53:55,699 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:53:55,699 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:53:55,969 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/eylvpie0-media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 18:53:56,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:53:56,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 18:53:56,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:53:56,633 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:56,645 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/aqskltpv-media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 18:53:57,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:57,629 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:58,103 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:53:58,634 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:53:59,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:00,269 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:00,637 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:01,104 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:01,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:01,496 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:01,632 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:02,632 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:03,105 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:03,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:04,640 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:05,447 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:05,645 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:05,927 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:54:05,927 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:54:05,928 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:54:06,106 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:54:06,106 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:54:06,360 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/aczj1xtw-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:54:06,641 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:07,107 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:07,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:08,654 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:09,107 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:09,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:10,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:10,782 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:11,108 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:11,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:12,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:13,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:14,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:15,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:16,108 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:54:16,108 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:16,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:16,475 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:16,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:17,110 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:17,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:18,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:19,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:20,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:21,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:21,643 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:22,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:23,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:24,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:25,644 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:26,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:26,645 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:27,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:28,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:29,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:30,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:31,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:31,476 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:31,598 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:31,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:32,646 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:33,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:34,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:35,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:36,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:37,307 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:37,654 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:37,918 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:54:37,918 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4 with policy now
+2025-01-18 18:54:37,919 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:54:37,919 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:54:37,919 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:54:37,922 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:54:38,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:54:38,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 18:54:38,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:54:38,208 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/i4z1pkjl-media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 18:54:38,490 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/tal85z4g-media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 18:54:38,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:39,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:39,648 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:40,118 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:40,654 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:41,663 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:42,510 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:42,672 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:43,119 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:43,664 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:44,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:45,120 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:45,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:46,108 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:54:46,493 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:54:46,493 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:54:46,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:47,619 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:47,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:48,672 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:49,684 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:49,982 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:54:49,982 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:54:49,982 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:54:50,122 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:54:50,122 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:54:50,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:51,122 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:51,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:52,672 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:53,123 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:53,512 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:53,671 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:54,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:55,124 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:54:55,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:56,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:57,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:58,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:54:59,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:54:59,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:00,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:01,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:01,480 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:01,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:02,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:03,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:04,556 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:04,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:05,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:06,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:07,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:08,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:09,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:10,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:10,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:11,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:12,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:13,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:14,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:15,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:15,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:16,108 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:55:16,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:16,479 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:16,669 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:17,132 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:17,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:18,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:19,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:20,543 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:20,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:21,676 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:21,988 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:55:21,988 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4 with policy now
+2025-01-18 18:55:21,991 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:55:21,991 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:55:21,991 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:55:21,992 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:55:22,134 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:55:22,134 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 18:55:22,135 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:55:22,251 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/9ibw7cch-media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 18:55:22,546 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/tbylwsyz-media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 18:55:22,674 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:23,135 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:23,673 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:24,135 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:24,689 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:25,554 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:25,681 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:26,685 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:27,136 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:27,680 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:28,677 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:29,137 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:29,686 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:30,690 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:31,029 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:55:31,030 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:55:31,030 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:31,030 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:55:31,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:31,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:55:31,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:55:31,394 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/uh15q6l5-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:55:31,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:31,491 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:31,690 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:32,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:33,138 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:33,692 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:34,690 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:35,139 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:35,691 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:36,466 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:36,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:37,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:38,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:39,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:40,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:41,505 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:41,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:42,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:43,142 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:43,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:44,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:45,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:46,109 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:55:46,480 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:55:46,480 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:55:46,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:46,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:47,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:48,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:49,145 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:55:49,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:50,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:51,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:52,506 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:52,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:53,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:54,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:55,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:56,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:57,507 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:55:57,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:58,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:55:59,695 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:00,696 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:01,149 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:01,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:01,495 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:01,700 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:02,362 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:56:02,362 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4 with policy now
+2025-01-18 18:56:02,371 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:56:02,379 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:56:02,379 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:56:02,379 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:56:02,627 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/sluts8c9-media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 18:56:02,703 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 18:56:03,150 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:56:03,431 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:03,465 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/cw47cria-media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 18:56:03,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:04,722 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:05,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:06,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:07,151 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:07,723 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:08,535 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:08,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:09,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:10,720 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:11,153 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:11,744 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:12,080 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:56:12,080 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:56:12,081 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:56:12,153 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:56:12,154 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:56:12,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:13,154 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:13,677 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:13,742 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:14,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:15,155 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:15,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:16,109 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:56:16,489 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:16,489 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:16,737 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:17,155 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:17,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:18,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:19,320 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:19,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:20,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:21,157 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:21,729 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:22,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:23,158 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:23,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:24,517 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:24,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:25,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:26,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:27,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:28,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:29,160 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:29,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:30,013 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:30,730 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:31,161 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:31,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:31,484 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:31,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:32,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:33,162 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:33,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:34,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:35,225 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:35,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:36,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:37,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:38,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:39,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:40,518 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:40,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:41,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:42,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:43,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:44,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:45,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:46,110 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:56:46,111 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:46,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:56:46,484 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:56:46,745 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:47,167 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:47,653 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:56:47,653 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4 with policy now
+2025-01-18 18:56:47,663 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:56:47,664 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:56:47,664 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:56:47,664 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:56:47,738 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:47,924 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fjt46xbn-media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 18:56:48,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:56:48,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 18:56:48,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:56:48,477 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/y14eyeb0-media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 18:56:48,770 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:49,168 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:49,769 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:50,763 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:51,137 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:51,761 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:52,774 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:53,169 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:53,767 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:54,782 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:55,170 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:56:55,767 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:56,145 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:56:56,766 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:57,781 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:58,778 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:56:59,783 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:00,783 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:01,479 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:01,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:01,522 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:01,785 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:02,138 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:57:02,138 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:57:02,139 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:57:02,173 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:57:02,173 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:57:02,472 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/uczda84y-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:57:02,790 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:03,173 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:03,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:04,813 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:05,174 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:05,788 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:06,527 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:06,817 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:07,174 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:07,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:08,818 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:09,811 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:10,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:11,176 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:11,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:12,523 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:12,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:13,807 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:14,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:15,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:16,110 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:57:16,484 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:16,485 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:16,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:17,178 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:17,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:17,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:18,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:19,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:20,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:21,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:22,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:23,523 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:23,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:24,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:25,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:26,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:27,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:28,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:29,523 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:29,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:30,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:31,485 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:31,485 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:31,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:32,809 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:33,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:34,540 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:34,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:35,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:36,810 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:37,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:38,567 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:57:38,568 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4 with policy now
+2025-01-18 18:57:38,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:57:38,594 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:57:38,594 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:57:38,595 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:57:38,822 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:38,851 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/34mx1zxu-media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 18:57:39,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:57:39,516 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/1qdkvnax-media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 18:57:39,628 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:39,833 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:40,187 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:40,843 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:41,833 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:42,864 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:43,188 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:43,854 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:44,855 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:45,253 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:45,856 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:46,111 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:57:46,497 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:57:46,497 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:57:46,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:47,190 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:47,857 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:48,852 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:49,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:49,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:57:49,197 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:57:49,197 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:57:49,198 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:57:49,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:50,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:57:50,701 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:50,879 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:51,191 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:51,883 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:52,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:53,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:54,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:55,193 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:57:55,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:56,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:57:56,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:57,865 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:58,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:57:59,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:00,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:01,488 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:01,488 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:01,631 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:01,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:02,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:03,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:04,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:05,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:06,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:07,552 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:07,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:08,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:09,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:10,867 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:11,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:12,554 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:12,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:13,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:14,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:15,868 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:16,111 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:58:16,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:16,513 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:16,869 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:17,201 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:17,624 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:17,869 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:18,870 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:19,889 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:20,876 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:21,148 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:58:21,148 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4 with policy now
+2025-01-18 18:58:21,152 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:58:21,152 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:58:21,153 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:58:21,153 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:58:21,203 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:21,203 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:58:21,203 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 18:58:21,204 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:58:21,430 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ehpxwp7a-media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 18:58:21,767 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/w390kric-media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 18:58:21,901 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:22,889 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:23,204 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:23,558 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:23,896 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:24,891 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:25,891 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:26,909 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:27,205 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:27,941 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:28,920 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:28,942 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:29,206 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:29,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:30,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:31,207 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:58:31,249 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:58:31,250 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:58:31,250 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:58:31,513 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:31,513 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:31,928 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:32,207 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:58:32,208 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:58:32,473 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/coyho2su-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:58:32,912 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:33,208 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:33,949 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:34,305 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:34,920 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:35,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:36,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:37,209 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:37,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:38,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:39,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:40,117 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:40,913 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:41,211 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:58:41,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:42,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:43,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:44,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:45,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:45,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:46,112 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:58:46,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:58:46,499 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:58:46,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:47,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:48,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:49,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:50,596 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:50,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:51,914 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:52,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:53,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:54,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:55,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:56,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:58:56,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:57,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:58,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:58:59,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:00,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:01,219 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:01,499 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:01,499 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:01,595 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:01,915 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:02,917 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:03,220 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:03,823 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:59:03,823 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4 with policy now
+2025-01-18 18:59:03,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:59:03,826 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:03,826 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:03,827 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:03,922 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:04,107 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/awwdnehm-media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 18:59:04,220 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:04,220 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:04,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 18:59:04,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:59:04,604 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/4qny7a6b-media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 18:59:04,968 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:05,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:05,935 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:06,221 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:06,950 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:07,382 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:07,952 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:08,947 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:09,222 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:09,958 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:10,940 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:11,223 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:11,961 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:12,585 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:12,949 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:13,224 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:13,224 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:59:13,304 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:13,304 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:13,305 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:13,993 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:14,224 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:14,974 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:15,225 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:15,972 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:16,112 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:59:16,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:16,537 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:16,972 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:17,952 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:18,467 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:18,952 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:19,227 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:19,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:20,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:21,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:22,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:23,576 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:23,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:24,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:25,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:26,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:27,230 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:27,953 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:28,576 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:28,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:29,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:30,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:31,510 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:31,510 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:31,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:32,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:33,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:33,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:34,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:35,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:36,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:37,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:38,954 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:39,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:39,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:40,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:41,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:42,955 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:43,956 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:44,963 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:45,021 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:45,238 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:45,554 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 18:59:45,554 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4 with policy now
+2025-01-18 18:59:45,588 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 18:59:45,588 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:45,588 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:45,589 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:45,859 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ofnyy30v-media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 18:59:45,963 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:46,112 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 18:59:46,238 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:46,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:46,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 18:59:46,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 18:59:46,537 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 18:59:46,537 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 18:59:46,594 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ooipvqyr-media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 18:59:46,958 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:47,239 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:47,969 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:48,963 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:49,971 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:50,117 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:50,964 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:51,241 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:51,987 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:52,988 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:53,242 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:53,994 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:54,983 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:55,297 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 18:59:55,989 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:56,243 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 18:59:56,356 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 18:59:56,357 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 18:59:56,357 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 18:59:57,008 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:57,243 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 18:59:57,244 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 18:59:57,984 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:59,013 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 18:59:59,244 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:00,008 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:00,993 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:01,107 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:01,245 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:01,543 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:01,543 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:02,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:03,007 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:04,005 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:05,008 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:06,011 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:06,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:07,039 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:08,014 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:09,014 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:09,248 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:10,014 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:11,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:12,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:12,593 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:13,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:14,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:15,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:16,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:16,113 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:00:16,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:16,521 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:17,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:17,634 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:18,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:19,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:20,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:21,015 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:22,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:23,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:23,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:24,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:25,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:26,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:27,016 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:28,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:29,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:29,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:30,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:31,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:31,521 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:31,521 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:32,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:33,017 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:34,018 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:35,021 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:35,042 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:35,793 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:00:35,793 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4 with policy now
+2025-01-18 19:00:35,805 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:00:35,806 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:00:35,806 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:00:35,806 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:00:36,026 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:36,126 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/cnkxpgbz-media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:00:36,259 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:00:36,628 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/1c5mhm1s-media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:00:37,048 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:38,048 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:38,260 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:39,057 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:40,063 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:40,105 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:41,044 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:42,082 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:42,261 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:43,063 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:44,078 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:44,262 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:45,078 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:45,182 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:46,095 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:46,113 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:00:46,539 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:00:46,539 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:00:47,076 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:48,088 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:49,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:50,090 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:50,264 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:00:50,422 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:00:50,422 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:00:50,422 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:50,422 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:00:50,523 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/rssavz5p-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:00:51,122 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:51,265 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:00:52,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:52,266 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:53,107 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:54,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:55,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:56,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:56,268 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:00:56,314 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:00:57,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:58,100 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:00:59,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:00,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:01,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:01,533 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:01,534 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:01,670 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:02,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:03,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:04,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:05,101 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:06,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:07,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:07,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:08,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:09,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:10,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:11,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:12,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:12,607 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:13,102 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:14,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:15,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:16,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:16,114 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:01:16,532 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:16,533 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:17,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:17,691 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:18,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:19,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:20,103 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:21,147 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:22,120 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:22,279 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:22,644 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:01:22,644 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4 with policy now
+2025-01-18 19:01:22,658 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:01:22,659 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:01:22,659 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:01:22,659 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:01:22,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:22,949 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/1t8esl9o-media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:01:23,134 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:23,279 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:01:23,279 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:01:23,280 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:01:23,634 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/hfjrq3td-media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:01:24,127 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:24,280 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:25,158 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:26,164 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:27,138 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:28,170 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:28,281 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:28,642 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:29,138 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:30,144 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:30,282 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:01:30,472 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:01:30,472 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:01:30,474 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:01:31,158 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:31,283 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:01:31,540 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:31,540 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:32,162 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:32,283 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:33,163 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:34,067 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:34,163 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:35,185 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:36,162 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:36,285 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:37,170 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:38,175 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:39,196 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:39,628 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:40,193 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:41,186 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:42,222 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:43,234 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:44,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:44,288 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:45,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:45,475 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:46,114 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:01:46,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:46,289 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:01:46,538 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:01:46,538 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:01:47,187 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:48,188 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:49,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:50,188 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:50,659 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:51,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:52,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:53,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:54,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:55,189 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:56,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:56,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:01:57,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:58,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:01:59,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:00,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:01,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:01,538 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:01,539 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:01,692 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:02,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:03,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:04,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:05,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:06,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:07,190 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:07,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:08,191 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:09,191 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:10,229 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:11,211 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:11,927 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:02:11,927 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4 with policy now
+2025-01-18 19:02:11,937 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:02:11,938 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:02:11,938 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:02:11,938 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:02:12,200 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:12,246 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/juj99v43-media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:02:12,300 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:02:12,663 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/tq3lyu01-media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:02:12,987 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:13,218 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:14,233 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:14,301 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:15,227 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:16,115 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:02:16,224 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:16,585 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:16,586 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:17,227 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:18,274 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:18,302 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:18,777 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:19,231 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:20,289 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:20,303 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:21,267 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:22,268 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:23,279 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:24,295 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:24,649 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:25,293 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:26,263 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:27,290 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:28,291 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:29,283 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:30,282 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:30,647 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:31,286 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:31,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:31,583 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:32,283 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:33,285 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:34,317 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:35,314 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:35,709 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:36,292 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:37,316 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:38,315 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:39,308 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:40,311 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:40,719 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:41,341 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:42,320 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:43,333 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:44,371 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:45,357 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:46,115 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:02:46,117 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:46,378 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:46,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:02:46,588 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:02:47,438 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:48,389 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:49,394 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:50,316 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:02:50,378 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:50,566 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:02:50,566 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:02:50,566 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:02:51,304 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:51,316 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:02:51,403 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:52,317 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:52,391 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:53,384 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:54,317 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:54,408 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:55,410 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:56,318 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:02:56,407 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:56,529 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:02:57,447 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:58,413 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:02:59,437 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:00,446 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:01,446 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:01,610 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:01,611 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:01,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:02,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:03,466 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:04,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:05,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:06,323 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:06,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:07,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:07,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:08,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:09,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:10,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:11,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:12,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:12,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:13,441 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:14,326 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:14,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:15,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:16,116 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:03:16,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:16,572 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:16,574 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:17,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:17,678 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:18,442 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:19,445 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:20,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:21,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:22,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:23,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:23,667 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:24,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:25,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:26,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:27,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:28,332 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:28,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:28,668 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:29,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:30,444 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:31,449 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:31,581 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:31,582 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:32,334 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:32,473 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:32,549 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:03:32,549 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:03:32,549 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4 with policy now
+2025-01-18 19:03:32,550 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:03:32,550 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:03:32,550 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:03:32,861 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/9fl1agn2-media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:03:33,334 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:03:33,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:03:33,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:03:33,476 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:33,709 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/zm6gx6cm-media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:03:34,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:34,478 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:34,618 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:35,335 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:35,490 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:36,501 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:37,464 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:38,337 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:38,495 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:39,468 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:40,353 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:40,470 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:41,487 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:42,338 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:03:42,508 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:43,516 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:44,525 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:45,519 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:45,698 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:46,116 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:03:46,527 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:46,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:03:46,588 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:03:47,524 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:48,581 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:49,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:50,529 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:50,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:51,545 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:52,561 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:53,548 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:54,567 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:55,551 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:56,577 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:56,694 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:03:57,584 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:58,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:03:59,587 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:00,592 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:01,592 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:01,594 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:01,595 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:01,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:02,598 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:03,590 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:04,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:05,645 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:06,618 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:06,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:07,629 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:08,349 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:04:08,609 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fon0epps-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:04:08,637 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:08,659 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:04:08,659 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:04:08,660 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:04:09,350 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:04:09,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:10,350 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:10,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:11,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:11,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:12,351 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:12,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:13,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:14,352 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:14,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:15,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:16,116 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:04:16,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:16,579 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:16,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:17,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:17,714 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:18,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:19,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:20,620 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:21,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:22,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:23,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:23,704 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:24,356 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:24,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:25,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:26,357 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:26,621 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:27,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:28,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:28,704 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:29,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:30,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:31,579 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:31,579 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:31,622 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:32,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:33,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:33,704 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:34,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:35,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:36,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:37,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:38,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:39,428 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:39,623 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:40,636 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:40,978 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:04:40,978 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4 with policy now
+2025-01-18 19:04:40,999 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:04:40,999 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:04:40,999 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:04:41,000 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:04:41,262 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ruqorwuy-media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:04:41,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:04:41,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:04:41,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:04:41,639 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:41,690 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/fxtr8vke-media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:04:42,364 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:42,710 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:43,365 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:43,641 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:44,656 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:44,684 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:45,686 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:46,117 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:04:46,366 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:46,583 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:04:46,584 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:04:46,651 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:47,664 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:48,666 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:49,662 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:50,053 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:50,368 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:04:50,662 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:51,653 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:52,660 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:53,700 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:54,691 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:55,682 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:55,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:04:56,710 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:57,675 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:58,707 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:59,372 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:04:59,721 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:04:59,721 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:04:59,721 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:04:59,722 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:05:00,372 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:00,372 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:05:00,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:01,165 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:01,608 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:01,609 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:01,712 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:02,373 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:02,776 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:03,693 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:04,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:05,744 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:06,375 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:06,533 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:06,724 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:07,737 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:08,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:09,731 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:10,737 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:11,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:11,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:12,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:13,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:14,378 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:14,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:15,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:16,117 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:05:16,379 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:16,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:16,580 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:16,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:17,725 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:17,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:18,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:19,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:20,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:21,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:22,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:23,717 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:23,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:24,726 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:25,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:26,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:27,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:28,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:29,717 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:29,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:30,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:31,580 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:31,580 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:31,727 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:32,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:33,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:34,721 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:34,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:35,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:36,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:37,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:38,728 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:39,733 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:40,063 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:40,393 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:40,670 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:05:40,670 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4 with policy now
+2025-01-18 19:05:40,679 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:05:40,679 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:05:40,679 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:05:40,679 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:05:40,734 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:40,981 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/5fk8t884-media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:05:41,390 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:05:41,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:05:41,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:05:41,743 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/c81m63pf-media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:05:41,761 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:42,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:42,811 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:43,391 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:43,804 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:44,770 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:45,305 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:45,782 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:46,118 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:05:46,393 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:46,601 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:05:46,601 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:05:46,785 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:47,754 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:48,759 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:49,764 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:50,315 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:50,394 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:50,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:05:50,759 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:50,780 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:05:50,780 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:05:50,781 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:05:51,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:05:51,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:05:51,771 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:52,395 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:52,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:53,789 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:54,396 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:54,788 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:55,808 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:55,961 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:05:56,397 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:05:56,820 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:57,796 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:58,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:05:59,830 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:00,813 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:01,613 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:01,613 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:01,732 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:01,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:02,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:03,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:04,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:05,825 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:06,734 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:06,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:07,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:08,402 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:08,798 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:09,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:10,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:11,734 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:11,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:12,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:13,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:14,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:15,799 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:16,118 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:06:16,596 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:16,596 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:16,735 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:16,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:17,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:18,801 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:19,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:20,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:21,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:21,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:22,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:23,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:24,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:25,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:26,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:26,802 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:27,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:28,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:29,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:30,412 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:30,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:31,596 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:31,596 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:31,736 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:31,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:32,803 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:33,885 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:34,657 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:06:34,658 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4 with policy now
+2025-01-18 19:06:34,675 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:06:34,675 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:34,675 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:34,676 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:34,813 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:34,977 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/ea2nxp9j-media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:06:35,414 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:35,414 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:06:35,414 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval
+2025-01-18 19:06:35,773 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/e8qe9gzx-media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:06:35,828 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:36,415 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:36,752 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:36,851 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:37,416 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:37,845 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:38,851 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:39,826 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:40,418 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:40,846 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:41,775 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:41,858 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:42,846 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:43,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:43,844 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:43,844 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:43,844 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:43,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:44,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:44,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:44,420 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:44,848 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:44,890 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-18 19:06:44,891 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: log_artifact
+2025-01-18 19:06:45,390 INFO    wandb-upload_0:2553803 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa2smyjth
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval
+2025-01-18 19:06:45,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table
+2025-01-18 19:06:45,491 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:06:45,777 INFO    SenderThread:2553803 [sender.py:send_request_log_artifact():1525] logged artifact run-q1ti958r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODI5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAwNzM4NQ==', 'latestArtifact': None}}
+2025-01-18 19:06:45,777 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:06:45,777 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json with policy now
+2025-01-18 19:06:45,779 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:45,779 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,780 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,783 DEBUG   SenderThread:2553803 [sender.py:send():379] send: telemetry
+2025-01-18 19:06:45,784 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,785 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,785 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,786 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,787 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,788 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,789 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,789 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,789 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,790 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,791 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,791 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,792 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,792 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,796 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,796 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,797 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,797 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,800 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,800 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,801 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:45,801 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:45,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:45,873 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-18 19:06:45,873 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: log_artifact
+2025-01-18 19:06:45,917 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-18 19:06:45,965 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/3wfnb2p7-media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:06:46,224 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/w508hlfq-media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:06:46,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:46,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:46,421 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:46,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:06:46,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval
+2025-01-18 19:06:46,426 INFO    wandb-upload_3:2553803 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4cq89eih
+2025-01-18 19:06:46,597 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-18 19:06:46,608 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/jcv4qu0w-media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:06:46,726 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/xbeut6o6-events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:06:46,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:46,871 INFO    SenderThread:2553803 [sender.py:send_request_log_artifact():1525] logged artifact run-q1ti958r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODI5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAwNzM4Nw==', 'latestArtifact': None}}
+2025-01-18 19:06:46,871 DEBUG   SenderThread:2553803 [sender.py:send():379] send: files
+2025-01-18 19:06:46,871 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json with policy now
+2025-01-18 19:06:46,871 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:06:46,871 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:06:46,872 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:46,872 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:46,872 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:06:46,873 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: stop_status
+2025-01-18 19:06:47,127 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /tmp/tmpyc1v30mzwandb/zai78lqq-media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:06:47,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:06:47,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:48,422 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:06:48,849 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:49,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:50,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:51,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:52,747 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:52,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:53,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:54,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:55,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:56,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:57,850 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:58,747 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:06:58,851 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:06:59,813 DEBUG   SenderThread:2553803 [sender.py:send():379] send: exit
+2025-01-18 19:06:59,813 INFO    SenderThread:2553803 [sender.py:send_exit():586] handling exit code: 0
+2025-01-18 19:06:59,814 INFO    SenderThread:2553803 [sender.py:send_exit():588] handling runtime: 1093
+2025-01-18 19:06:59,814 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:06:59,814 INFO    SenderThread:2553803 [sender.py:send_exit():594] send defer
+2025-01-18 19:06:59,814 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,814 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-18 19:06:59,814 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 1
+2025-01-18 19:06:59,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-18 19:06:59,815 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-18 19:06:59,815 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 2
+2025-01-18 19:06:59,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-18 19:06:59,815 DEBUG   SystemMonitor:2553803 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-18 19:06:59,815 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-18 19:06:59,816 DEBUG   SystemMonitor:2553803 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-18 19:06:59,816 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined disk monitor
+2025-01-18 19:06:59,865 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-18 19:06:59,865 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined memory monitor
+2025-01-18 19:06:59,865 INFO    HandlerThread:2553803 [interfaces.py:finish():200] Joined network monitor
+2025-01-18 19:06:59,865 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,865 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-18 19:06:59,866 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 3
+2025-01-18 19:06:59,866 DEBUG   SenderThread:2553803 [sender.py:send():379] send: stats
+2025-01-18 19:06:59,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,866 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-18 19:06:59,866 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:06:59,866 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-18 19:06:59,866 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 4
+2025-01-18 19:06:59,866 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:06:59,866 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-18 19:07:00,428 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:07:02,430 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:07:06,852 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:06,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:07:06,853 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-18 19:07:06,853 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,854 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-18 19:07:06,854 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 5
+2025-01-18 19:07:06,854 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:06,854 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:06,854 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-18 19:07:06,854 DEBUG   SenderThread:2553803 [sender.py:send():379] send: history
+2025-01-18 19:07:06,855 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: summary_record
+2025-01-18 19:07:06,855 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:07:06,856 DEBUG   SenderThread:2553803 [sender.py:send():379] send: summary
+2025-01-18 19:07:06,860 INFO    SenderThread:2553803 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-18 19:07:06,860 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,860 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-18 19:07:06,860 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 6
+2025-01-18 19:07:06,860 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:06,861 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-18 19:07:06,861 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,861 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-18 19:07:06,864 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: status_report
+2025-01-18 19:07:06,992 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 7
+2025-01-18 19:07:06,993 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:06,993 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-18 19:07:06,993 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:06,993 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-18 19:07:07,432 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:07:07,432 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 19:07:07,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:08,555 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 8
+2025-01-18 19:07:08,556 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:08,556 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:08,556 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-18 19:07:08,556 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:08,556 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-18 19:07:08,556 INFO    SenderThread:2553803 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-18 19:07:08,559 INFO    SenderThread:2553803 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-18 19:07:08,561 INFO    SenderThread:2553803 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-18 19:07:08,570 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 9
+2025-01-18 19:07:08,571 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:08,571 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-18 19:07:08,571 DEBUG   SenderThread:2553803 [sender.py:send():379] send: artifact
+2025-01-18 19:07:08,815 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:09,284 INFO    wandb-upload_0:2553803 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzx8vfrwt
+2025-01-18 19:07:09,284 INFO    wandb-upload_3:2553803 [upload_job.py:push():85] Skipped uploading /tmp/tmpsv0ptkxm/wandb-job.json
+2025-01-18 19:07:09,392 INFO    wandb-upload_2:2553803 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpcvsay24k
+2025-01-18 19:07:09,433 INFO    Thread-12 :2553803 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:07:10,017 INFO    SenderThread:2553803 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODU1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTQwMTA5Mw==', 'versionIndex': 445}}}
+2025-01-18 19:07:10,017 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,017 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-18 19:07:10,017 INFO    SenderThread:2553803 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-18 19:07:10,434 INFO    SenderThread:2553803 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files
+2025-01-18 19:07:10,434 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml config.yaml
+2025-01-18 19:07:10,434 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json wandb-summary.json
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt requirements.txt
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-metadata.json wandb-metadata.json
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log output.log
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0 events.out.tfevents.1737244126.leonmkim-ROG-Strix-G15CS-G15CS.2553479.0
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/diff.patch diff.patch
+2025-01-18 19:07:10,435 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4 media/videos/eval/1_eval_1_66c73c1e8035e9955d13.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4 media/videos/eval/8_eval_8_c7ac84718a1584e3ff20.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4 media/videos/eval/12_eval_12_815bf7acfa2781cd7af5.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4 media/videos/eval/7_eval_7_21e28c4cb1d06fc94de3.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4 media/videos/eval/17_eval_17_97651cd5baa14e04e160.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4 media/videos/eval/4_eval_4_ae4f4aa5127f311e839c.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4 media/videos/eval/10_eval_10_9f1bedc7d64acfd93348.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4 media/videos/eval/9_eval_9_a1569ddaa0ccfb738c03.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4 media/videos/eval/2_eval_2_d91598e5051835f677e8.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4 media/videos/eval/14_eval_14_015a814176a13d9d5eb6.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4 media/videos/eval/0_eval_0_9c4b549c276868919c0f.mp4
+2025-01-18 19:07:10,441 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4 media/videos/eval/16_eval_16_9b69f164b68c214c2794.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4 media/videos/eval/11_eval_11_603d199abb1a8aa26ee3.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4 media/videos/eval/3_eval_3_6706618b69c2f852f270.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4 media/videos/eval/13_eval_13_be54cbdf6a2c74d5872d.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4 media/videos/eval/18_eval_18_7b8fe7251cc74d7e3aaa.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4 media/videos/eval/19_eval_19_2c92e633fda071abe0bf.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4 media/videos/eval/6_eval_6_cccf0749ac898e59a2f9.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4 media/videos/eval/5_eval_5_15c189fb02c615acea53.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4 media/videos/eval/15_eval_15_fddbe9cae0cebc69c5e8.mp4
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json media/table/eval/total_success_rate_ci_21_9ef68d5d87d1210dec34.table.json
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json media/table/eval/success_rate_ci_20_5376ac71dfa7920aa911.table.json
+2025-01-18 19:07:10,442 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 10
+2025-01-18 19:07:10,442 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:10,443 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:10,443 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-18 19:07:10,443 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,443 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-18 19:07:10,443 INFO    SenderThread:2553803 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-18 19:07:10,615 INFO    wandb-upload_3:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/requirements.txt
+2025-01-18 19:07:10,636 INFO    wandb-upload_0:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/wandb-summary.json
+2025-01-18 19:07:10,644 INFO    wandb-upload_1:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/config.yaml
+2025-01-18 19:07:10,688 INFO    wandb-upload_2:2553803 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/files/output.log
+2025-01-18 19:07:10,816 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:10,816 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:10,889 INFO    Thread-11 (_thread_body):2553803 [sender.py:transition_state():614] send defer: 11
+2025-01-18 19:07:10,889 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:10,889 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-18 19:07:10,889 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,889 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-18 19:07:10,889 INFO    SenderThread:2553803 [file_pusher.py:join():175] waiting for file pusher
+2025-01-18 19:07:10,892 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 12
+2025-01-18 19:07:10,892 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:10,893 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-18 19:07:10,893 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:10,893 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-18 19:07:10,893 INFO    SenderThread:2553803 [file_stream.py:finish():601] file stream finish called
+2025-01-18 19:07:11,080 INFO    SenderThread:2553803 [file_stream.py:finish():605] file stream finish is done
+2025-01-18 19:07:11,080 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 13
+2025-01-18 19:07:11,080 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:11,080 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-18 19:07:11,080 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:11,080 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-18 19:07:11,081 INFO    SenderThread:2553803 [sender.py:transition_state():614] send defer: 14
+2025-01-18 19:07:11,081 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: defer
+2025-01-18 19:07:11,081 DEBUG   SenderThread:2553803 [sender.py:send():379] send: final
+2025-01-18 19:07:11,081 INFO    HandlerThread:2553803 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-18 19:07:11,081 DEBUG   SenderThread:2553803 [sender.py:send():379] send: footer
+2025-01-18 19:07:11,081 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: defer
+2025-01-18 19:07:11,081 INFO    SenderThread:2553803 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-18 19:07:11,081 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:11,082 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:11,082 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: server_info
+2025-01-18 19:07:11,082 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-18 19:07:11,082 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: server_info
+2025-01-18 19:07:11,084 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-18 19:07:11,085 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-18 19:07:11,086 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-18 19:07:11,129 DEBUG   SenderThread:2553803 [sender.py:send_request():406] send_request: poll_exit
+2025-01-18 19:07:11,130 INFO    MainThread:2553803 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-18 19:07:11,130 INFO    MainThread:2553803 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-18 19:07:11,131 INFO    MainThread:2553803 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-18 19:07:11,131 DEBUG   HandlerThread:2553803 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-18 19:07:11,131 INFO    HandlerThread:2553803 [handler.py:finish():882] shutting down handler
+2025-01-18 19:07:12,084 INFO    WriterThread:2553803 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb
+2025-01-18 19:07:12,130 INFO    SenderThread:2553803 [sender.py:finish():1615] shutting down sender
+2025-01-18 19:07:12,130 INFO    SenderThread:2553803 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-18 19:07:12,130 INFO    SenderThread:2553803 [file_pusher.py:join():175] waiting for file pusher
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug.log b/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..4885733bddd8dfce240f49cdbec9179dae64f66b
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Configure stats pid to 2553479
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/settings
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug.log
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/wandb/run-20250118_184845-q1ti958r/logs/debug-internal.log
+2025-01-18 18:48:45,735 INFO    MainThread:2553479 [wandb_init.py:init():569] calling init triggers
+2025-01-18 18:48:45,736 INFO    MainThread:2553479 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r'}
+2025-01-18 18:48:45,736 INFO    MainThread:2553479 [wandb_init.py:init():619] starting backend
+2025-01-18 18:48:45,736 INFO    MainThread:2553479 [wandb_init.py:init():623] setting up manager
+2025-01-18 18:48:45,740 INFO    MainThread:2553479 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-18 18:48:45,741 INFO    MainThread:2553479 [wandb_init.py:init():631] backend started and connected
+2025-01-18 18:48:45,753 INFO    MainThread:2553479 [wandb_init.py:init():720] updated telemetry
+2025-01-18 18:48:45,762 INFO    MainThread:2553479 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-18 18:48:46,018 INFO    MainThread:2553479 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-18 18:48:46,090 INFO    MainThread:2553479 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-18 18:48:46,090 INFO    MainThread:2553479 [wandb_init.py:init():804] starting run threads in backend
+2025-01-18 18:48:46,471 INFO    MainThread:2553479 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-18 18:48:46,471 INFO    MainThread:2553479 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-18 18:48:46,472 INFO    MainThread:2553479 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-18 18:48:46,472 INFO    MainThread:2553479 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-18 18:48:46,474 INFO    MainThread:2553479 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-18 18:48:46,474 INFO    MainThread:2553479 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/q1ti958r/tb, True
+2025-01-18 18:50:24,759 INFO    MainThread:2553479 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-18 19:07:12,132 WARNING MsgRouterThr:2553479 [router.py:message_loop():77] message_loop has been closed
diff --git a/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb b/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..71f4065b7dbaedd30048f00ceab44d3c9b584837
--- /dev/null
+++ b/q1ti958r/wandb/run-20250118_184845-q1ti958r/run-q1ti958r.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:704ce55ac2fa3ab53441f7d87c7f30913c051b01499bf5b5fd05e982248c4262
+size 246738
diff --git a/rgws6h04/.hydra/config.yaml b/rgws6h04/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6f829134d57914397f938447dc055c8c1721578e
--- /dev/null
+++ b/rgws6h04/.hydra/config.yaml
@@ -0,0 +1,352 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '3465_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 7500
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      crop_distractors_tlhw: null
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+      crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw}
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/rgws6h04/.hydra/hydra.yaml b/rgws6h04/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..112e042aab7742d488fe774d4ca42564f3775f9f
--- /dev/null
+++ b/rgws6h04/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/rgws6h04/.hydra/overrides.yaml b/rgws6h04/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/rgws6h04/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/rgws6h04/episode_rosbags/aligned_depth_to_color_K.npy b/rgws6h04/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/rgws6h04/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/rgws6h04/episode_rosbags/cam_tf_world.npy b/rgws6h04/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/rgws6h04/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/rgws6h04/episode_rosbags/color_K.npy b/rgws6h04/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/rgws6h04/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/rgws6h04/episode_rosbags/depth_K.npy b/rgws6h04/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/rgws6h04/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/rgws6h04/episode_rosbags/episode_0_2025-01-21-20-46-39.bag b/rgws6h04/episode_rosbags/episode_0_2025-01-21-20-46-39.bag
new file mode 100644
index 0000000000000000000000000000000000000000..8a4dd45903f0c025cd2953e06bb350e89bb810cd
--- /dev/null
+++ b/rgws6h04/episode_rosbags/episode_0_2025-01-21-20-46-39.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1844b21878363546a4242de2411c136ce3e031126a0284b4a9dc460e5b4cf503
+size 1408860652
diff --git a/rgws6h04/eval_robot.log b/rgws6h04/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..c9101211aadcb1bc1e9266b9091e2a2080252ae5
--- /dev/null
+++ b/rgws6h04/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-21 20:46:15,793][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-21 20:46:15,795][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/rgws6h04/eval_video/0_eval.mp4 b/rgws6h04/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..62a9ea3c203506d02e9c175c13ded046ef767ff2
--- /dev/null
+++ b/rgws6h04/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6984eb6a23a5c4d44097934333e5e9d5b359106e35a01ba3b2cec87142613546
+size 1171764
diff --git a/rgws6h04/eval_video/1_eval.mp4 b/rgws6h04/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4c4aeebbdf51c5677e0dbae0b34d4cbbdfd31878
--- /dev/null
+++ b/rgws6h04/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf3f5901b54201219032d4eb210f3d277a1b4dc5f2082f2ff9e71098af6d41a
+size 1181972
diff --git a/rgws6h04/eval_video/2_eval.mp4 b/rgws6h04/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4c596c2998fe29b29fec75cbbcdf0683bcca0f17
--- /dev/null
+++ b/rgws6h04/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c3370287ca4ba8df1e31322fe8bfaa35d98b871885c05c3fb824ad5bdabae80
+size 1212946
diff --git a/rgws6h04/eval_video/3_eval.mp4 b/rgws6h04/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3ba706c12eb9d29f39d9a59ffc0011f2109d676c
--- /dev/null
+++ b/rgws6h04/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf4a29f3f357c4d6846e3c5cd6dd618a7d20baa1997c679c98d1914c57e3493
+size 1271708
diff --git a/rgws6h04/eval_video/4_eval.mp4 b/rgws6h04/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..486b64271508c22dc4763ac630b7d44a4e4dbd70
--- /dev/null
+++ b/rgws6h04/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73de45e70fef794b02e098f336c1cda4ca7831867af77dde4d0ff752da6cc481
+size 1253273
diff --git a/rgws6h04/tb/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0 b/rgws6h04/tb/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
new file mode 100644
index 0000000000000000000000000000000000000000..0ed33af478f1068eae66fda925c3e6af385cf2ac
--- /dev/null
+++ b/rgws6h04/tb/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72ad29e52ceaeefe4a999de5fb3dc48b8c1a0b990e1cdf3b3b35368187769b4f
+size 6539
diff --git a/rgws6h04/wandb/debug-internal.log b/rgws6h04/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..d4e8c7fdad2948e1d738ad89c7c1f66e988260a5
--- /dev/null
+++ b/rgws6h04/wandb/debug-internal.log
@@ -0,0 +1,736 @@
+2025-01-21 20:46:21,770 INFO    StreamThr :2640811 [internal.py:wandb_internal():85] W&B internal server running at pid: 2640811, started at: 2025-01-21 20:46:21.769718
+2025-01-21 20:46:21,771 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status
+2025-01-21 20:46:21,781 INFO    WriterThread:2640811 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb
+2025-01-21 20:46:21,782 DEBUG   SenderThread:2640811 [sender.py:send():379] send: header
+2025-01-21 20:46:21,789 DEBUG   SenderThread:2640811 [sender.py:send():379] send: run
+2025-01-21 20:46:22,100 INFO    SenderThread:2640811 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files
+2025-01-21 20:46:22,100 INFO    SenderThread:2640811 [sender.py:_start_run_threads():1188] run started: rgws6h04 with start time 1737510381.769739
+2025-01-21 20:46:22,117 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: check_version
+2025-01-21 20:46:22,117 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: check_version
+2025-01-21 20:46:22,209 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: run_start
+2025-01-21 20:46:22,214 DEBUG   HandlerThread:2640811 [system_info.py:__init__():26] System info init
+2025-01-21 20:46:22,214 DEBUG   HandlerThread:2640811 [system_info.py:__init__():41] System info init done
+2025-01-21 20:46:22,214 INFO    HandlerThread:2640811 [system_monitor.py:start():194] Starting system monitor
+2025-01-21 20:46:22,214 INFO    SystemMonitor:2640811 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-21 20:46:22,214 INFO    HandlerThread:2640811 [system_monitor.py:probe():214] Collecting system info
+2025-01-21 20:46:22,215 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started cpu monitoring
+2025-01-21 20:46:22,215 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started disk monitoring
+2025-01-21 20:46:22,216 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started gpu monitoring
+2025-01-21 20:46:22,217 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started memory monitoring
+2025-01-21 20:46:22,217 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started network monitoring
+2025-01-21 20:46:22,227 DEBUG   HandlerThread:2640811 [system_info.py:probe():152] Probing system
+2025-01-21 20:46:22,229 DEBUG   HandlerThread:2640811 [system_info.py:_probe_git():137] Probing git
+2025-01-21 20:46:22,233 DEBUG   HandlerThread:2640811 [system_info.py:_probe_git():145] Probing git done
+2025-01-21 20:46:22,234 DEBUG   HandlerThread:2640811 [system_info.py:probe():200] Probing system done
+2025-01-21 20:46:22,234 DEBUG   HandlerThread:2640811 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:46:22.227637', 'startedAt': '2025-01-22T01:46:21.760646', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.2005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.269, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.017, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 315.79178619384766}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-21 20:46:22,234 INFO    HandlerThread:2640811 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-21 20:46:22,234 INFO    HandlerThread:2640811 [system_monitor.py:probe():227] Publishing system info
+2025-01-21 20:46:22,234 DEBUG   HandlerThread:2640811 [system_info.py:_save_code():44] Saving code
+2025-01-21 20:46:22,235 DEBUG   HandlerThread:2640811 [system_info.py:_save_code():67] Saving code done
+2025-01-21 20:46:22,235 DEBUG   HandlerThread:2640811 [system_info.py:_save_patches():84] Saving git patches
+2025-01-21 20:46:22,297 DEBUG   HandlerThread:2640811 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-21 20:46:22,297 INFO    HandlerThread:2640811 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-21 20:46:22,300 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:46:22,300 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-21 20:46:22,301 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-21 20:46:22,301 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-21 20:46:22,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-21 20:46:22,527 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: python_packages
+2025-01-21 20:46:22,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:46:22,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:46:22,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:22,531 INFO    HandlerThread:2640811 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/tb"
+  save: true
+}
+control {
+  relay_id: "3b79abafd7c147dbb67b62c5ccec0411"
+}
+_info {
+  stream_id: "rgws6h04"
+}
+
+2025-01-21 20:46:22,531 DEBUG   HandlerThread:2640811 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-21 20:46:22,567 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/em9b4p0m-wandb-metadata.json
+2025-01-21 20:46:22,568 INFO    wandb-upload_2:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/bkez637d-diff.patch
+2025-01-21 20:46:22,618 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/ggjq7ilr-code/FISH/eval_robot.py
+2025-01-21 20:46:22,685 DEBUG   SenderThread:2640811 [sender.py:send():379] send: telemetry
+2025-01-21 20:46:22,685 DEBUG   SenderThread:2640811 [sender.py:send():379] send: tbrecord
+2025-01-21 20:46:22,686 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:46:22,686 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0 with policy live
+2025-01-21 20:46:22,877 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/j59rvv0o-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:23,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch
+2025-01-21 20:46:23,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH
+2025-01-21 20:46:23,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code
+2025-01-21 20:46:23,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:24,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:25,101 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:25,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:26,109 DEBUG   SenderThread:2640811 [sender.py:send():379] send: config
+2025-01-21 20:46:26,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:27,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:27,116 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:27,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:28,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:29,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:29,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:30,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:31,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:31,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:32,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:32,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:33,104 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:33,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:34,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:35,104 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:35,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:36,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:37,105 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:37,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:46:37,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:46:37,569 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:37,661 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:38,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:39,106 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:39,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:40,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:41,106 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:41,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:42,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:43,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:43,625 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:44,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:45,107 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:45,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:46,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:47,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:48,533 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:49,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:49,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:50,545 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:51,545 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:52,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:46:52,527 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:46:52,573 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:53,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:54,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:54,660 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:55,110 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
+2025-01-21 20:46:55,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:56,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:57,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:58,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:59,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:00,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:00,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:01,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:02,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:03,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:04,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:05,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:05,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:06,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:07,234 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:47:07,234 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4 with policy now
+2025-01-21 20:47:07,234 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:47:07,235 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:47:07,235 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:47:07,236 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:47:07,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:07,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:07,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:07,603 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/ruylq4md-media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:47:08,450 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/52gi6y36-media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:47:08,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:09,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:09,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:10,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:10,627 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:11,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:12,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:13,116 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:13,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:14,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:15,116 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:15,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:15,649 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:16,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:17,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:18,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:19,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:20,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:21,276 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:21,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:22,119 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:47:22,217 DEBUG   SystemMonitor:2640811 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-21 20:47:22,218 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:47:22,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:22,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:22,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:23,119 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:23,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:24,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:25,120 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:25,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:26,284 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:26,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:27,121 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:27,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:28,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:29,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:30,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:31,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:31,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:32,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:33,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:34,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:35,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:36,533 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:36,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:37,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:37,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:37,569 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:38,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:39,552 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:40,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:41,125 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:41,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:42,057 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:42,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:43,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:44,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:45,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:46,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:47,073 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:47,127 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:47,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:48,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:49,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:50,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:51,128 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:51,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:52,218 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:47:52,219 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:52,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:52,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:52,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:53,129 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:53,399 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:47:53,399 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4 with policy now
+2025-01-21 20:47:53,404 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:47:53,405 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:47:53,405 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:47:53,406 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:47:53,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:53,668 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/uc8ana8g-media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:47:54,129 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:47:54,129 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:47:54,130 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:47:54,466 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/ajmxmasf-media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:47:54,558 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:55,130 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:55,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:56,130 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:56,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:57,534 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:57,561 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:58,557 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:59,131 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:59,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:00,574 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:01,131 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:01,559 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:02,536 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:02,566 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:03,561 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:03,923 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:48:03,924 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:48:03,924 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:48:04,133 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:48:04,133 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:48:04,397 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/yxfnrmja-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:48:04,601 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:05,133 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:05,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:06,562 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:07,134 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:07,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:07,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:07,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:07,673 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:08,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:09,134 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:09,710 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:10,710 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:11,135 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:11,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:12,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:13,534 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:13,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:14,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:15,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:16,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:17,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:18,534 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:18,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:19,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:20,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:21,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:22,219 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:48:22,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:22,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:22,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:23,701 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:23,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:24,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:25,140 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:25,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:26,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:27,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:28,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:29,535 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:29,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:30,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:31,714 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:32,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:33,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:34,714 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:35,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:35,715 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:36,559 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:48:36,559 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4 with policy now
+2025-01-21 20:48:36,561 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:48:36,562 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:48:36,562 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:48:36,563 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:48:36,716 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:36,837 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/91jv94yr-media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:48:37,454 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/mhp55w33-media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:48:37,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:37,531 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:37,722 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:38,721 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:39,145 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:39,725 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:40,725 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:41,145 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:41,237 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:41,724 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:42,728 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:43,725 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:44,739 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:45,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:45,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:48:45,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:45,752 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:48:45,753 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:48:45,772 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:48:46,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:48:46,737 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:46,997 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:47,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:47,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:48,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:49,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:50,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:51,149 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:51,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:52,219 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:48:52,220 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:52,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:52,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:52,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:53,149 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:53,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:54,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:55,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:56,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:57,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:57,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:58,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:59,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:00,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:01,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:02,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:02,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:03,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:04,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:05,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:06,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:07,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:07,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:07,634 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:07,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:08,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:09,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:10,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:11,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:12,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:13,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:13,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:14,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:15,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:16,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:17,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:18,739 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:18,844 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:19,110 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:49:19,110 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4 with policy now
+2025-01-21 20:49:19,118 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:49:19,118 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:49:19,118 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:49:19,119 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:49:19,158 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:49:19,158 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:19,159 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:49:19,159 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:49:19,388 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/9wlo9n3o-media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:49:19,684 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/uurugayp-media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:49:19,744 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:20,742 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:21,159 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:21,742 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:22,220 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:49:22,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:22,533 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:22,743 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:23,748 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:24,756 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:24,801 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:25,160 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:25,747 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:26,749 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:27,161 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:27,749 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:28,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:29,162 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:49:29,443 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/mj7cf4oa-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:49:29,750 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:29,806 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:49:29,806 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:49:29,806 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:29,806 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:49:30,162 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:49:30,752 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:31,162 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:31,752 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:32,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:33,761 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:34,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:35,164 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:35,213 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:35,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:36,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:37,165 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:37,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:37,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:37,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:38,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:39,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:40,618 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:40,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:41,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:42,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:43,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:44,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:45,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:46,543 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:46,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:47,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:48,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:49,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:50,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:51,543 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:51,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:52,220 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:49:52,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:52,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:52,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:53,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:54,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:55,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:56,681 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:56,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:57,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:58,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:59,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:00,756 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:01,756 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:02,051 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:02,768 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:03,105 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:50:03,105 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4 with policy now
+2025-01-21 20:50:03,111 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:50:03,111 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:03,111 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:03,112 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:03,173 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:03,173 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:03,173 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:03,174 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:50:03,378 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/94y2vecq-media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:03,684 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/vok2cnu7-media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:03,765 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:04,774 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:05,174 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:05,764 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:06,809 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:07,544 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:50:07,544 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:50:07,727 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:07,778 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:08,773 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:09,175 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:09,778 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:10,777 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:11,176 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:11,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:12,773 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:12,864 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:12,864 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:12,865 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:12,865 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:13,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:13,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:13,500 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:50:13,500 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:50:13,698 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:50:13,771 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:14,020 INFO    wandb-upload_1:2640811 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmg_f16mn
+2025-01-21 20:50:14,108 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:50:14,119 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:50:14,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:14,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:14,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:14,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media
+2025-01-21 20:50:14,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval
+2025-01-21 20:50:14,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table
+2025-01-21 20:50:14,625 INFO    SenderThread:2640811 [sender.py:send_request_log_artifact():1525] logged artifact run-rgws6h04-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTAzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NzQzOQ==', 'latestArtifact': None}}
+2025-01-21 20:50:14,625 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:50:14,625 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,626 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send():379] send: telemetry
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,630 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,630 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,631 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,631 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,631 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,631 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,632 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,632 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,632 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,632 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,633 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,633 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,633 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,633 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,633 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,634 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,634 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,634 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,634 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,635 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,635 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,635 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:50:14,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:14,813 INFO    wandb-upload_2:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/phfifvxm-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:14,815 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/qgzh22eb-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:15,045 INFO    wandb-upload_2:2640811 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbvqhghg_
+2025-01-21 20:50:15,082 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/q978auoe-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:15,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:15,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:15,495 INFO    SenderThread:2640811 [sender.py:send_request_log_artifact():1525] logged artifact run-rgws6h04-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTA1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NzQ0Nw==', 'latestArtifact': None}}
+2025-01-21 20:50:15,495 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:50:15,495 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json with policy now
+2025-01-21 20:50:15,496 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:15,496 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:15,496 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:15,757 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/s3jgt8fr-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:15,774 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:16,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:16,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:17,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:17,773 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:18,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:18,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:19,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:20,249 DEBUG   SenderThread:2640811 [sender.py:send():379] send: exit
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_exit():586] handling exit code: 0
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_exit():588] handling runtime: 238
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_exit():594] send defer
+2025-01-21 20:50:20,250 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,250 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-21 20:50:20,250 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 1
+2025-01-21 20:50:20,251 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-21 20:50:20,251 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,251 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-21 20:50:20,251 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 2
+2025-01-21 20:50:20,251 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-21 20:50:20,251 DEBUG   SystemMonitor:2640811 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined disk monitor
+2025-01-21 20:50:20,252 DEBUG   SystemMonitor:2640811 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-21 20:50:20,303 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-21 20:50:20,304 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined memory monitor
+2025-01-21 20:50:20,304 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined network monitor
+2025-01-21 20:50:20,304 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,304 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-21 20:50:20,304 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 3
+2025-01-21 20:50:20,304 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:50:20,304 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,305 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-21 20:50:20,305 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,305 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-21 20:50:20,305 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 4
+2025-01-21 20:50:20,305 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,305 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-21 20:50:21,180 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:21,180 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:26,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:26,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:26,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,873 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-21 20:50:26,873 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 5
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:26,874 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:26,874 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-21 20:50:26,875 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:26,875 DEBUG   SenderThread:2640811 [sender.py:send():379] send: summary
+2025-01-21 20:50:26,879 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:26,879 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,879 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-21 20:50:26,879 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 6
+2025-01-21 20:50:26,879 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:26,879 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-21 20:50:26,879 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,880 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-21 20:50:26,883 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:26,995 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 7
+2025-01-21 20:50:26,995 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:26,996 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-21 20:50:26,996 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,996 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-21 20:50:27,182 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:27,183 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
+2025-01-21 20:50:27,251 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:29,467 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 8
+2025-01-21 20:50:29,467 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:29,468 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:29,468 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-21 20:50:29,468 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:29,468 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-21 20:50:29,468 INFO    SenderThread:2640811 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-21 20:50:29,471 INFO    SenderThread:2640811 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-21 20:50:29,472 INFO    SenderThread:2640811 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-21 20:50:29,486 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 9
+2025-01-21 20:50:29,486 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:29,486 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-21 20:50:29,487 DEBUG   SenderThread:2640811 [sender.py:send():379] send: artifact
+2025-01-21 20:50:29,892 INFO    wandb-upload_2:2640811 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz5ivij6l
+2025-01-21 20:50:30,002 INFO    wandb-upload_1:2640811 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu4ne52gz
+2025-01-21 20:50:30,021 INFO    wandb-upload_0:2640811 [upload_job.py:push():88] Uploaded file /tmp/tmpa66g1g3_/wandb-job.json
+2025-01-21 20:50:30,184 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:30,252 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:30,872 INFO    SenderThread:2640811 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'versionIndex': 453}}}
+2025-01-21 20:50:30,872 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:30,873 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-21 20:50:30,873 INFO    SenderThread:2640811 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-21 20:50:31,184 INFO    SenderThread:2640811 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml config.yaml
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0 events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json wandb-summary.json
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt requirements.txt
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json wandb-metadata.json
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log output.log
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch diff.patch
+2025-01-21 20:50:31,187 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-21 20:50:31,187 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4 media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:50:31,188 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4 media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:50:31,189 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4 media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:50:31,189 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4 media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:50:31,194 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4 media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:31,195 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:31,195 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:31,196 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 10
+2025-01-21 20:50:31,196 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,196 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,196 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-21 20:50:31,196 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,196 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-21 20:50:31,196 INFO    SenderThread:2640811 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:50:31,253 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:31,253 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,374 INFO    wandb-upload_2:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
+2025-01-21 20:50:31,413 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:31,424 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt
+2025-01-21 20:50:31,425 INFO    wandb-upload_4:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:31,452 INFO    wandb-upload_3:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/0dnpg8u8-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:31,653 INFO    Thread-11 (_thread_body):2640811 [sender.py:transition_state():614] send defer: 11
+2025-01-21 20:50:31,653 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,653 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-21 20:50:31,653 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,653 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-21 20:50:31,653 INFO    SenderThread:2640811 [file_pusher.py:join():175] waiting for file pusher
+2025-01-21 20:50:31,654 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 12
+2025-01-21 20:50:31,654 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,655 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-21 20:50:31,655 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,655 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-21 20:50:31,655 INFO    SenderThread:2640811 [file_stream.py:finish():601] file stream finish called
+2025-01-21 20:50:31,867 INFO    SenderThread:2640811 [file_stream.py:finish():605] file stream finish is done
+2025-01-21 20:50:31,867 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 13
+2025-01-21 20:50:31,867 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,867 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-21 20:50:31,867 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,867 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-21 20:50:31,868 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 14
+2025-01-21 20:50:31,868 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,868 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-21 20:50:31,868 DEBUG   SenderThread:2640811 [sender.py:send():379] send: final
+2025-01-21 20:50:31,868 DEBUG   SenderThread:2640811 [sender.py:send():379] send: footer
+2025-01-21 20:50:31,868 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,868 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-21 20:50:31,868 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:31,869 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,869 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: server_info
+2025-01-21 20:50:31,869 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:31,870 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: server_info
+2025-01-21 20:50:31,871 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-21 20:50:31,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-21 20:50:31,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:31,935 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,936 INFO    MainThread:2640811 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-21 20:50:31,936 INFO    MainThread:2640811 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-21 20:50:31,936 INFO    MainThread:2640811 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-21 20:50:31,937 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-21 20:50:31,937 INFO    HandlerThread:2640811 [handler.py:finish():882] shutting down handler
+2025-01-21 20:50:32,870 INFO    WriterThread:2640811 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb
+2025-01-21 20:50:32,936 INFO    SenderThread:2640811 [sender.py:finish():1615] shutting down sender
+2025-01-21 20:50:32,936 INFO    SenderThread:2640811 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:50:32,936 INFO    SenderThread:2640811 [file_pusher.py:join():175] waiting for file pusher
diff --git a/rgws6h04/wandb/debug.log b/rgws6h04/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..ce02924333803090c8bcd5c18ae546a5f51016cf
--- /dev/null
+++ b/rgws6h04/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Configure stats pid to 2640629
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/settings
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug.log
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug-internal.log
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():569] calling init triggers
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04'}
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():619] starting backend
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():623] setting up manager
+2025-01-21 20:46:21,768 INFO    MainThread:2640629 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-21 20:46:21,769 INFO    MainThread:2640629 [wandb_init.py:init():631] backend started and connected
+2025-01-21 20:46:21,781 INFO    MainThread:2640629 [wandb_init.py:init():720] updated telemetry
+2025-01-21 20:46:21,787 INFO    MainThread:2640629 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-21 20:46:22,116 INFO    MainThread:2640629 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-21 20:46:22,155 INFO    MainThread:2640629 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-21 20:46:22,156 INFO    MainThread:2640629 [wandb_init.py:init():804] starting run threads in backend
+2025-01-21 20:46:22,528 INFO    MainThread:2640629 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-21 20:46:22,528 INFO    MainThread:2640629 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-21 20:46:22,528 INFO    MainThread:2640629 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-21 20:46:22,529 INFO    MainThread:2640629 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-21 20:46:22,530 INFO    MainThread:2640629 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-21 20:46:22,531 INFO    MainThread:2640629 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/tb, True
+2025-01-21 20:46:26,108 INFO    MainThread:2640629 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-21 20:50:32,937 WARNING MsgRouterThr:2640629 [router.py:message_loop():77] message_loop has been closed
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py
@@ -0,0 +1,606 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config), 
+				# hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				# self.cfg.agent.config.policy_cfg.input_shapes,
+				# hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..976d65f62519b7f561214fe172696b7e3277c9f3
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
@@ -0,0 +1,927 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '3465_0'
+wandb_notes:
+  desc: null
+  value: 3465_0_restarted_1
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: false
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+restore_missing_checkpoint:
+  desc: null
+  value: null
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        crop_distractors_tlhw:
+        - 32
+        - 68
+        - 174
+        - 174
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: false
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+        crop_distractors_tlhw: null
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          crop_distractors_tlhw:
+          - 32
+          - 68
+          - 174
+          - 174
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+test:
+  desc: null
+  value:
+    username: leonmkim
+    hostname: grasp-login1
+    name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+    root_dir: ~/fish_leon
+    expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+    expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+    semantic_demo_grouping_name: semantic_demo_grouping.yaml
+    semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+    include_groups_list: all
+    batch_size: 128
+    store_dataset_in_memory: false
+    use_tb: true
+    local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup
+    local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1
+    resume_wandb_run: false
+feature_type:
+  desc: null
+  value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 7500
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1737510381
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..8da10724284d70d8fca0dedf572670b3116372fc
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch
@@ -0,0 +1,28 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb5fa99..e77ad21 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17
+ 
+ # Number of evaluation trajectories
+ # num_eval: 10
+-num_eval: 20
++num_eval: 5
+ 
+ debug_timestamps: False
+ open_loop: False
+@@ -139,12 +139,12 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 06/20 demos per book
+ # crp_D+mask+act history
+-# wandb_run_id: '3465_0'
++wandb_run_id: '3465_0'
+ # wandb_run_id: '3465_1' # seed 1
+ # wandb_run_id: '3465_2' # seed 2
+ 
+ # crp_D+mask+contact+act history
+-wandb_run_id: '4295_0'
++# wandb_run_id: '4295_0'
+ # wandb_run_id: '4295_1' # seed 1
+ # wandb_run_id: '4295_2' # seed 2
+ 
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..f762859d82639dec5477727bf2e6532a4206426b
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[43, 45, 0.9555555555555556, 0.8341377376841794, 0.9955952054761087, 0.86772614710129, 0.9944009267667571, 0.8845370262092178, 0.9872875529298147, 0.8906857277416631, 0.9872872819149455]]}
\ No newline at end of file
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4 b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..62a9ea3c203506d02e9c175c13ded046ef767ff2
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6984eb6a23a5c4d44097934333e5e9d5b359106e35a01ba3b2cec87142613546
+size 1171764
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4 b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4c4aeebbdf51c5677e0dbae0b34d4cbbdfd31878
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf3f5901b54201219032d4eb210f3d277a1b4dc5f2082f2ff9e71098af6d41a
+size 1181972
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4 b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4c596c2998fe29b29fec75cbbcdf0683bcca0f17
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c3370287ca4ba8df1e31322fe8bfaa35d98b871885c05c3fb824ad5bdabae80
+size 1212946
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4 b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3ba706c12eb9d29f39d9a59ffc0011f2109d676c
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf4a29f3f357c4d6846e3c5cd6dd618a7d20baa1997c679c98d1914c57e3493
+size 1271708
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4 b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..486b64271508c22dc4763ac630b7d44a4e4dbd70
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73de45e70fef794b02e098f336c1cda4ca7831867af77dde4d0ff752da6cc481
+size 1253273
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..9f1eb7baaebb1667ea56c24074c28cb3f4f673d7
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
@@ -0,0 +1,268 @@
+
+loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1737510386.109211]: resetting environment
+[INFO] [1737510386.112611]: cleared current plan
+[INFO] [1737510386.113767]: moving to home
+[INFO] [1737510387.116082]: reached home
+[INFO] [1737510387.117536]: reset action history
+[INFO] [1737510389.824857]: environment reset
+Starting episode 0
+[INFO] [1737510389.826594]: resetting environment
+[INFO] [1737510389.830330]: cleared current plan
+[INFO] [1737510389.831224]: moving to home
+[INFO] [1737510390.833337]: reached home
+[INFO] [1737510390.834658]: reset action history
+[INFO] [1737510393.542351]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1737510395.484667]: resetting environment
+[INFO] [1737510395.489720]: cleared current plan
+[INFO] [1737510395.490943]: moving to home
+[INFO] [1737510396.493144]: reached home
+[INFO] [1737510396.494837]: reset action history
+[INFO] [1737510399.252956]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/episode_rosbags/episode_0_2025-01-21-20-46-39.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2640629.log
+started roslaunch server http://158.130.50.37:36507/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2640912]
+process[print_text-2]: started with pid [2640936]
+process[rosbag_record-3]: started with pid [2640937]
+started bagging!
+For topic gripper_width: timestamp difference is 105487179 for nearest: 1737510402579215359 - target: 1737510402473728180 at idx 46
+[INFO] [1737510426.288207]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1737510427.011994]: Stored episode 1.
+[INFO] [1737510427.012249]: Saving video...
+[INFO] [1737510427.222461]: Video saved!
+Starting episode 1
+[INFO] [1737510427.235871]: resetting environment
+[INFO] [1737510427.241355]: cleared current plan
+[INFO] [1737510427.241594]: moving to home
+[INFO] [1737510430.844176]: reached home
+[INFO] [1737510430.844658]: reset action history
+[INFO] [1737510433.606613]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510441.282536]: resetting environment
+[INFO] [1737510441.287676]: cleared current plan
+[INFO] [1737510441.287992]: moving to home
+[INFO] [1737510442.289289]: reached home
+[INFO] [1737510442.290425]: reset action history
+[INFO] [1737510444.997974]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/episode_rosbags/episode_1_2025-01-21-20-47-24.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2640629.log
+started roslaunch server http://158.130.50.37:43393/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2641056]
+process[print_text-5]: started with pid [2641080]
+process[rosbag_record-6]: started with pid [2641081]
+started bagging!
+For topic gripper_width: timestamp difference is 101146167 for nearest: 1737510459012543147 - target: 1737510458911396980 at idx 181
+For topic gripper_width: timestamp difference is 100951321 for nearest: 1737510466012538321 - target: 1737510465911587000 at idx 249
+For topic gripper_width: timestamp difference is 103720393 for nearest: 1737510469412542786 - target: 1737510469308822393 at idx 249
+[INFO] [1737510471.998138]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1737510472.776285]: Stored episode 2.
+[INFO] [1737510472.776516]: Saving video...
+[INFO] [1737510473.374960]: Video saved!
+Starting episode 2
+[INFO] [1737510473.403248]: resetting environment
+[INFO] [1737510473.410917]: cleared current plan
+[INFO] [1737510473.411889]: moving to home
+[WARN] [1737510473.316577]: Plan exhausted
+[WARN] [1737510473.356900]: Plan exhausted
+[WARN] [1737510473.396766]: Plan exhausted
+[INFO] [1737510477.018608]: reached home
+[INFO] [1737510477.021305]: reset action history
+[INFO] [1737510479.792392]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510483.543850]: resetting environment
+[INFO] [1737510483.552281]: cleared current plan
+[INFO] [1737510483.553264]: moving to home
+[INFO] [1737510484.555328]: reached home
+[INFO] [1737510484.557204]: reset action history
+[INFO] [1737510487.332568]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/episode_rosbags/episode_2_2025-01-21-20-48-07.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2640629.log
+started roslaunch server http://158.130.50.37:45879/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2641217]
+process[print_text-8]: started with pid [2641218]
+process[rosbag_record-9]: started with pid [2641219]
+started bagging!
+For topic gripper_width: timestamp difference is -136267692 for nearest: 1737510502312511653 - target: 1737510502448779345 at idx 191
+[INFO] [1737510515.527957]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1737510516.303241]: Stored episode 3.
+[INFO] [1737510516.311700]: Saving video...
+[INFO] [1737510516.529573]: Video saved!
+Starting episode 3
+[INFO] [1737510516.565596]: resetting environment
+[INFO] [1737510516.584153]: cleared current plan
+[INFO] [1737510516.586600]: moving to home
+[INFO] [1737510520.190820]: reached home
+[INFO] [1737510520.194043]: reset action history
+[INFO] [1737510522.933839]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510524.929521]: resetting environment
+[INFO] [1737510524.948966]: cleared current plan
+[INFO] [1737510524.949319]: moving to home
+[INFO] [1737510525.950956]: reached home
+[INFO] [1737510525.952543]: reset action history
+[INFO] [1737510528.731694]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/episode_rosbags/episode_3_2025-01-21-20-48-48.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2640629.log
+started roslaunch server http://158.130.50.37:46369/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2641361]
+process[print_text-11]: started with pid [2641362]
+process[rosbag_record-12]: started with pid [2641363]
+started bagging!
+For topic gripper_width: timestamp difference is 107889897 for nearest: 1737510557012542254 - target: 1737510556904652357 at idx 249
+[INFO] [1737510558.007761]: Storing episode...
+[print_text-11] killing on exit[pub_text-10] killing on exit
+[rosbag_record-12] killing on exit
+[INFO] [1737510558.837490]: Stored episode 4.
+[INFO] [1737510558.845019]: Saving video...
+[INFO] [1737510559.071334]: Video saved!
+Starting episode 4
+[INFO] [1737510559.120319]: resetting environment
+[INFO] [1737510559.138125]: cleared current plan
+[INFO] [1737510559.141392]: moving to home
+[INFO] [1737510562.749077]: reached home
+[INFO] [1737510562.754578]: reset action history
+[INFO] [1737510565.531022]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1737510569.110580]: resetting environment
+[INFO] [1737510569.121973]: cleared current plan
+[INFO] [1737510569.125555]: moving to home
+[INFO] [1737510570.131629]: reached home
+[INFO] [1737510570.141769]: reset action history
+[INFO] [1737510572.874898]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/episode_rosbags/episode_4_2025-01-21-20-49-32.bag
+... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2640629.log
+started roslaunch server http://158.130.50.37:45859/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2641506]
+process[print_text-14]: started with pid [2641507]
+process[rosbag_record-15]: started with pid [2641508]
+started bagging!
+[INFO] [1737510602.050753]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1737510602.826487]: Stored episode 5.
+[INFO] [1737510602.835502]: Saving video...
+[INFO] [1737510603.070795]: Video saved!
+[INFO] [1737510603.112450]: resetting environment
+[INFO] [1737510603.137198]: cleared current plan
+[INFO] [1737510603.156104]: moving to home
+[INFO] [1737510606.764727]: reached home
+[INFO] [1737510606.771071]: reset action history
+[INFO] [1737510609.555914]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..21b78d5fadeed8de3547e029e9c0e964c370aa74
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-22T01:46:22.227637",
+    "startedAt": "2025-01-22T01:46:21.760646",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3561.2005,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4495.269,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.017,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 315.79178619384766
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..9769a00947810750c3b69339c8e962abc2685240
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "6984eb6a23a5c4d44097934333e5e9d5b359106e35a01ba3b2cec87142613546", "size": 1171764, "path": "media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4"}, "global_step": 665, "_timestamp": 1737510613.9976697, "_runtime": 232.22793078422546, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "3cf3f5901b54201219032d4eb210f3d277a1b4dc5f2082f2ff9e71098af6d41a", "size": 1181972, "path": "media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "3c3370287ca4ba8df1e31322fe8bfaa35d98b871885c05c3fb824ad5bdabae80", "size": 1212946, "path": "media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ddf4a29f3f357c4d6846e3c5cd6dd618a7d20baa1997c679c98d1914c57e3493", "size": 1271708, "path": "media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "73de45e70fef794b02e098f336c1cda4ca7831867af77dde4d0ff752da6cc481", "size": 1253273, "path": "media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://o253x1h70t9qj1vclgyusyjbqejttjfq9635bf5nqk6t1stm8wed8jgtbexckc2pndnj1jzeyk159pb2560qz4o705h1ze4gt2cwm0ac3lf8avynzjqd54sy9k0n3zdo/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://x5md07v7nt67h1e648fnx7ms4guz76dr1se648g3sa4opoqk1f2jtxkafvulqtl4xjqindg0efd824top2n6c1kn6vyphk2stbsgihwyfr1xa5h5jm4vw556hmwwtmc2:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 43, "total_num_episodes": 45, "total_success_rate": 0.9555555555555556, "total_success_rate_lb_0.01": 0.8341377376841794, "total_success_rate_ub_0.01": 0.9955952054761087, "total_success_rate_lb_0.025": 0.86772614710129, "total_success_rate_ub_0.025": 0.9944009267667571, "total_success_rate_lb_0.05": 0.8845370262092178, "total_success_rate_ub_0.05": 0.9872875529298147, "total_success_rate_lb_0.1": 0.8906857277416631, "total_success_rate_ub_0.1": 0.9872872819149455, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "05144a46373be58183546cfa7c55fac95736a76fe195d68fef0cdde89a47270b", "size": 518, "artifact_path": "wandb-client-artifact://y8q0iny8220kvcmv9xjbtdc98ngbkjmi8939eqgxeih8vkdk8wtflw5pvm7dccr7b3fadytvub2g97xv1whxxj955xxnw5p680tze8pdxi85xobszo7z9tworwukfvuv/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://1380nzzhmk85hdydb5egriap8o4w0x8ly10gclkbpmsp2zkxg9g1kxxjsc34nvjvbiqhax1r1vjwy8wxhcmj8mib0emr1hncao5lwx2fka1kzh64nx7lkpdjrt9dfj3j:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 238}, "eval/total_success_rate": 0.9555555582046509, "eval/total_success_rate_lb_0.01": 0.8341377377510071, "eval/total_success_rate_ub_0.01": 0.9955952167510986, "eval/total_success_rate_lb_0.025": 0.8677261471748352, "eval/total_success_rate_ub_0.025": 0.9944009184837341, "eval/total_success_rate_lb_0.05": 0.8845370411872864, "eval/total_success_rate_ub_0.05": 0.9872875809669495, "eval/total_success_rate_lb_0.1": 0.8906857371330261, "eval/total_success_rate_ub_0.1": 0.9872872829437256}
\ No newline at end of file
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug-internal.log b/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..d4e8c7fdad2948e1d738ad89c7c1f66e988260a5
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug-internal.log
@@ -0,0 +1,736 @@
+2025-01-21 20:46:21,770 INFO    StreamThr :2640811 [internal.py:wandb_internal():85] W&B internal server running at pid: 2640811, started at: 2025-01-21 20:46:21.769718
+2025-01-21 20:46:21,771 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status
+2025-01-21 20:46:21,781 INFO    WriterThread:2640811 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb
+2025-01-21 20:46:21,782 DEBUG   SenderThread:2640811 [sender.py:send():379] send: header
+2025-01-21 20:46:21,789 DEBUG   SenderThread:2640811 [sender.py:send():379] send: run
+2025-01-21 20:46:22,100 INFO    SenderThread:2640811 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files
+2025-01-21 20:46:22,100 INFO    SenderThread:2640811 [sender.py:_start_run_threads():1188] run started: rgws6h04 with start time 1737510381.769739
+2025-01-21 20:46:22,117 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: check_version
+2025-01-21 20:46:22,117 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: check_version
+2025-01-21 20:46:22,209 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: run_start
+2025-01-21 20:46:22,214 DEBUG   HandlerThread:2640811 [system_info.py:__init__():26] System info init
+2025-01-21 20:46:22,214 DEBUG   HandlerThread:2640811 [system_info.py:__init__():41] System info init done
+2025-01-21 20:46:22,214 INFO    HandlerThread:2640811 [system_monitor.py:start():194] Starting system monitor
+2025-01-21 20:46:22,214 INFO    SystemMonitor:2640811 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-21 20:46:22,214 INFO    HandlerThread:2640811 [system_monitor.py:probe():214] Collecting system info
+2025-01-21 20:46:22,215 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started cpu monitoring
+2025-01-21 20:46:22,215 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started disk monitoring
+2025-01-21 20:46:22,216 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started gpu monitoring
+2025-01-21 20:46:22,217 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started memory monitoring
+2025-01-21 20:46:22,217 INFO    SystemMonitor:2640811 [interfaces.py:start():188] Started network monitoring
+2025-01-21 20:46:22,227 DEBUG   HandlerThread:2640811 [system_info.py:probe():152] Probing system
+2025-01-21 20:46:22,229 DEBUG   HandlerThread:2640811 [system_info.py:_probe_git():137] Probing git
+2025-01-21 20:46:22,233 DEBUG   HandlerThread:2640811 [system_info.py:_probe_git():145] Probing git done
+2025-01-21 20:46:22,234 DEBUG   HandlerThread:2640811 [system_info.py:probe():200] Probing system done
+2025-01-21 20:46:22,234 DEBUG   HandlerThread:2640811 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:46:22.227637', 'startedAt': '2025-01-22T01:46:21.760646', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.2005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.269, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.017, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 315.79178619384766}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-21 20:46:22,234 INFO    HandlerThread:2640811 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-21 20:46:22,234 INFO    HandlerThread:2640811 [system_monitor.py:probe():227] Publishing system info
+2025-01-21 20:46:22,234 DEBUG   HandlerThread:2640811 [system_info.py:_save_code():44] Saving code
+2025-01-21 20:46:22,235 DEBUG   HandlerThread:2640811 [system_info.py:_save_code():67] Saving code done
+2025-01-21 20:46:22,235 DEBUG   HandlerThread:2640811 [system_info.py:_save_patches():84] Saving git patches
+2025-01-21 20:46:22,297 DEBUG   HandlerThread:2640811 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-21 20:46:22,297 INFO    HandlerThread:2640811 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-21 20:46:22,300 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:46:22,300 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-21 20:46:22,301 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-21 20:46:22,301 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-21 20:46:22,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-21 20:46:22,527 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: python_packages
+2025-01-21 20:46:22,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:46:22,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:46:22,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:22,531 INFO    HandlerThread:2640811 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/tb"
+  save: true
+}
+control {
+  relay_id: "3b79abafd7c147dbb67b62c5ccec0411"
+}
+_info {
+  stream_id: "rgws6h04"
+}
+
+2025-01-21 20:46:22,531 DEBUG   HandlerThread:2640811 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-21 20:46:22,567 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/em9b4p0m-wandb-metadata.json
+2025-01-21 20:46:22,568 INFO    wandb-upload_2:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/bkez637d-diff.patch
+2025-01-21 20:46:22,618 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/ggjq7ilr-code/FISH/eval_robot.py
+2025-01-21 20:46:22,685 DEBUG   SenderThread:2640811 [sender.py:send():379] send: telemetry
+2025-01-21 20:46:22,685 DEBUG   SenderThread:2640811 [sender.py:send():379] send: tbrecord
+2025-01-21 20:46:22,686 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:46:22,686 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0 with policy live
+2025-01-21 20:46:22,877 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/j59rvv0o-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:46:23,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:23,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch
+2025-01-21 20:46:23,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH
+2025-01-21 20:46:23,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code
+2025-01-21 20:46:23,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:24,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:25,101 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:25,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:26,109 DEBUG   SenderThread:2640811 [sender.py:send():379] send: config
+2025-01-21 20:46:26,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:27,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:27,116 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:27,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:28,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:29,102 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:29,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:30,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:31,103 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:31,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:32,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:32,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:33,104 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:33,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:34,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:35,104 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:35,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:36,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:37,105 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:37,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:46:37,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:46:37,569 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:37,661 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:38,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:39,106 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:39,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:40,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:41,106 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:41,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:42,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:43,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:43,625 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:44,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:45,107 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:46:45,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:46,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:47,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:48,533 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:49,530 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:49,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:50,545 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:51,545 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:52,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:46:52,527 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:46:52,573 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:53,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:54,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:54,660 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:46:55,110 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
+2025-01-21 20:46:55,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:56,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:57,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:58,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:46:59,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:00,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:00,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:01,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:02,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:03,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:04,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:05,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:05,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:06,547 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:07,234 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:47:07,234 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4 with policy now
+2025-01-21 20:47:07,234 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:47:07,235 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:47:07,235 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:47:07,236 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:47:07,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:07,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:07,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:07,603 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/ruylq4md-media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media
+2025-01-21 20:47:08,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:47:08,450 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/52gi6y36-media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:47:08,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:09,114 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:09,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:10,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:10,627 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:11,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:12,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:13,116 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:13,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:14,548 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:15,116 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:15,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:15,649 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:16,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:17,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:18,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:19,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:20,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:21,276 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:21,549 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:22,119 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:47:22,217 DEBUG   SystemMonitor:2640811 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-21 20:47:22,218 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:47:22,527 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:22,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:22,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:23,119 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:23,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:24,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:25,120 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:25,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:26,284 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:26,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:27,121 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:27,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:28,550 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:29,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:30,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:31,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:31,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:32,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:33,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:34,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:35,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:36,533 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:36,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:37,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:37,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:37,569 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:38,551 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:39,552 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:40,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:41,125 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:41,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:42,057 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:42,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:43,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:44,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:45,553 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:46,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:47,073 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:47,127 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:47,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:48,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:49,554 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:50,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:51,128 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:51,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:52,218 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:47:52,219 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:52,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:47:52,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:47:52,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:53,129 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:53,399 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:47:53,399 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4 with policy now
+2025-01-21 20:47:53,404 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:47:53,405 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:47:53,405 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:47:53,406 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:47:53,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:53,668 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/uc8ana8g-media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:47:54,129 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:47:54,129 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:47:54,130 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:47:54,466 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/ajmxmasf-media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:47:54,558 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:55,130 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:55,555 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:56,130 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:56,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:57,534 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:47:57,561 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:58,557 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:47:59,131 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:47:59,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:00,574 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:01,131 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:01,559 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:02,536 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:02,566 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:03,561 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:03,923 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:48:03,924 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:48:03,924 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:48:04,133 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:48:04,133 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:48:04,397 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/yxfnrmja-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:48:04,601 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:05,133 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:05,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:06,562 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:07,134 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:07,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:07,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:07,560 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:07,673 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:08,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:09,134 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:09,710 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:10,710 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:11,135 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:11,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:12,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:13,534 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:13,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:14,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:15,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:16,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:17,711 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:18,534 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:18,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:19,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:20,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:21,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:22,219 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:48:22,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:22,528 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:22,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:23,701 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:23,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:24,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:25,140 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:25,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:26,712 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:27,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:28,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:29,535 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:29,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:30,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:31,714 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:32,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:33,713 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:34,714 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:35,528 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:35,715 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:36,559 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:48:36,559 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4 with policy now
+2025-01-21 20:48:36,561 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:48:36,562 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:48:36,562 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:48:36,563 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:48:36,716 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:36,837 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/91jv94yr-media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:48:37,144 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:48:37,454 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/mhp55w33-media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:48:37,531 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:37,531 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:37,722 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:38,721 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:39,145 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:39,725 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:40,725 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:41,145 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:41,237 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:41,724 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:42,728 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:43,725 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:44,739 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:45,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:45,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:48:45,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:45,752 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:48:45,753 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:48:45,772 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:48:46,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:48:46,737 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:46,997 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:47,147 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:47,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:48,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:49,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:50,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:51,149 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:51,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:52,219 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:48:52,220 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:52,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:48:52,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:48:52,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:53,149 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:48:53,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:54,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:55,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:56,730 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:57,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:48:57,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:58,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:48:59,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:00,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:01,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:02,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:02,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:03,731 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:04,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:05,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:06,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:07,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:07,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:07,634 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:07,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:08,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:09,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:10,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:11,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:12,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:13,537 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:13,732 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:14,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:15,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:16,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:17,733 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:18,739 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:18,844 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:19,110 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:49:19,110 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4 with policy now
+2025-01-21 20:49:19,118 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:49:19,118 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:49:19,118 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:49:19,119 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:49:19,158 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:49:19,158 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:19,159 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:49:19,159 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:49:19,388 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/9wlo9n3o-media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:49:19,684 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/uurugayp-media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:49:19,744 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:20,742 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:21,159 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:21,742 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:22,220 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:49:22,532 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:22,533 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:22,743 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:23,748 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:24,756 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:24,801 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:25,160 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:25,747 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:26,749 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:27,161 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:27,749 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:28,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:29,162 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:49:29,443 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/mj7cf4oa-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:49:29,750 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:29,806 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:49:29,806 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:49:29,806 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:29,806 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:49:30,162 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:49:30,752 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:31,162 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:31,752 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:32,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:33,761 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:34,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:35,164 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:35,213 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:35,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:36,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:37,165 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:49:37,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:37,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:37,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:38,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:39,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:40,618 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:40,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:41,753 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:42,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:43,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:44,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:45,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:46,543 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:46,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:47,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:48,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:49,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:50,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:51,543 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:51,754 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:52,220 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:49:52,529 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:49:52,529 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:49:52,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:53,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:54,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:55,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:56,681 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:49:56,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:57,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:58,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:49:59,755 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:00,756 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:01,756 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:02,051 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:02,768 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:03,105 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:50:03,105 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4 with policy now
+2025-01-21 20:50:03,111 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:50:03,111 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:03,111 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:03,112 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:03,173 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:03,173 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:03,173 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:03,174 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval
+2025-01-21 20:50:03,378 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/94y2vecq-media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:03,684 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/vok2cnu7-media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:03,765 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:04,774 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:05,174 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:05,764 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:06,809 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:07,544 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-21 20:50:07,544 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: stop_status
+2025-01-21 20:50:07,727 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:07,778 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:08,773 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:09,175 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:09,778 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:10,777 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:11,176 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:11,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:12,773 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:12,864 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:12,864 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:12,865 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:12,865 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:13,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:13,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:13,500 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:50:13,500 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:50:13,698 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:50:13,771 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:14,020 INFO    wandb-upload_1:2640811 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmg_f16mn
+2025-01-21 20:50:14,108 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-21 20:50:14,119 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-21 20:50:14,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:14,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:14,177 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:14,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media
+2025-01-21 20:50:14,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval
+2025-01-21 20:50:14,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table
+2025-01-21 20:50:14,625 INFO    SenderThread:2640811 [sender.py:send_request_log_artifact():1525] logged artifact run-rgws6h04-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTAzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NzQzOQ==', 'latestArtifact': None}}
+2025-01-21 20:50:14,625 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:50:14,625 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,626 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send():379] send: telemetry
+2025-01-21 20:50:14,626 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,630 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,630 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,631 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,631 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,631 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,631 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,632 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,632 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,632 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,632 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,633 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,633 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,633 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,633 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,633 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,634 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,634 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,634 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,634 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,635 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:14,635 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:14,635 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: log_artifact
+2025-01-21 20:50:14,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:14,813 INFO    wandb-upload_2:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/phfifvxm-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:14,815 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/qgzh22eb-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:15,045 INFO    wandb-upload_2:2640811 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbvqhghg_
+2025-01-21 20:50:15,082 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/q978auoe-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:15,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:15,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:15,495 INFO    SenderThread:2640811 [sender.py:send_request_log_artifact():1525] logged artifact run-rgws6h04-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTA1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NzQ0Nw==', 'latestArtifact': None}}
+2025-01-21 20:50:15,495 DEBUG   SenderThread:2640811 [sender.py:send():379] send: files
+2025-01-21 20:50:15,495 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json with policy now
+2025-01-21 20:50:15,496 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:15,496 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:15,496 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:15,757 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/s3jgt8fr-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:15,774 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:16,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:16,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:17,178 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:17,773 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:18,546 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:18,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:19,772 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:20,249 DEBUG   SenderThread:2640811 [sender.py:send():379] send: exit
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_exit():586] handling exit code: 0
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_exit():588] handling runtime: 238
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_exit():594] send defer
+2025-01-21 20:50:20,250 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,250 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-21 20:50:20,250 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-21 20:50:20,250 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 1
+2025-01-21 20:50:20,251 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-21 20:50:20,251 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,251 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-21 20:50:20,251 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 2
+2025-01-21 20:50:20,251 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-21 20:50:20,251 DEBUG   SystemMonitor:2640811 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-21 20:50:20,251 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined disk monitor
+2025-01-21 20:50:20,252 DEBUG   SystemMonitor:2640811 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-21 20:50:20,303 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-21 20:50:20,304 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined memory monitor
+2025-01-21 20:50:20,304 INFO    HandlerThread:2640811 [interfaces.py:finish():200] Joined network monitor
+2025-01-21 20:50:20,304 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,304 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-21 20:50:20,304 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 3
+2025-01-21 20:50:20,304 DEBUG   SenderThread:2640811 [sender.py:send():379] send: stats
+2025-01-21 20:50:20,304 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,305 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-21 20:50:20,305 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:20,305 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-21 20:50:20,305 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 4
+2025-01-21 20:50:20,305 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:20,305 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-21 20:50:21,180 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:21,180 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:26,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:26,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:26,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,873 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-21 20:50:26,873 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 5
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send():379] send: history
+2025-01-21 20:50:26,873 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: summary_record
+2025-01-21 20:50:26,874 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:26,874 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-21 20:50:26,875 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:26,875 DEBUG   SenderThread:2640811 [sender.py:send():379] send: summary
+2025-01-21 20:50:26,879 INFO    SenderThread:2640811 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-21 20:50:26,879 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,879 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-21 20:50:26,879 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 6
+2025-01-21 20:50:26,879 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:26,879 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-21 20:50:26,879 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,880 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-21 20:50:26,883 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: status_report
+2025-01-21 20:50:26,995 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 7
+2025-01-21 20:50:26,995 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:26,996 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-21 20:50:26,996 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:26,996 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-21 20:50:27,182 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:27,183 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
+2025-01-21 20:50:27,251 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:29,467 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 8
+2025-01-21 20:50:29,467 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:29,468 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:29,468 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-21 20:50:29,468 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:29,468 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-21 20:50:29,468 INFO    SenderThread:2640811 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-21 20:50:29,471 INFO    SenderThread:2640811 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-21 20:50:29,472 INFO    SenderThread:2640811 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-21 20:50:29,486 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 9
+2025-01-21 20:50:29,486 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:29,486 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-21 20:50:29,487 DEBUG   SenderThread:2640811 [sender.py:send():379] send: artifact
+2025-01-21 20:50:29,892 INFO    wandb-upload_2:2640811 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz5ivij6l
+2025-01-21 20:50:30,002 INFO    wandb-upload_1:2640811 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu4ne52gz
+2025-01-21 20:50:30,021 INFO    wandb-upload_0:2640811 [upload_job.py:push():88] Uploaded file /tmp/tmpa66g1g3_/wandb-job.json
+2025-01-21 20:50:30,184 INFO    Thread-12 :2640811 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:30,252 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:30,872 INFO    SenderThread:2640811 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'versionIndex': 453}}}
+2025-01-21 20:50:30,872 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:30,873 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-21 20:50:30,873 INFO    SenderThread:2640811 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-21 20:50:31,184 INFO    SenderThread:2640811 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml config.yaml
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0 events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json wandb-summary.json
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt requirements.txt
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-metadata.json wandb-metadata.json
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log output.log
+2025-01-21 20:50:31,185 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/diff.patch diff.patch
+2025-01-21 20:50:31,187 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-21 20:50:31,187 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4 media/videos/eval/1_eval_1_3cf3f5901b5420121903.mp4
+2025-01-21 20:50:31,188 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4 media/videos/eval/0_eval_0_6984eb6a23a5c4d44097.mp4
+2025-01-21 20:50:31,189 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4 media/videos/eval/2_eval_2_3c3370287ca4ba8df1e3.mp4
+2025-01-21 20:50:31,189 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4 media/videos/eval/3_eval_3_ddf4a29f3f357c4d6846.mp4
+2025-01-21 20:50:31,194 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4 media/videos/eval/4_eval_4_73de45e70fef794b02e0.mp4
+2025-01-21 20:50:31,195 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json
+2025-01-21 20:50:31,195 INFO    SenderThread:2640811 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-21 20:50:31,196 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 10
+2025-01-21 20:50:31,196 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,196 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,196 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-21 20:50:31,196 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,196 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-21 20:50:31,196 INFO    SenderThread:2640811 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:50:31,253 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:31,253 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,374 INFO    wandb-upload_2:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/config.yaml
+2025-01-21 20:50:31,413 INFO    wandb-upload_0:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/wandb-summary.json
+2025-01-21 20:50:31,424 INFO    wandb-upload_1:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/requirements.txt
+2025-01-21 20:50:31,425 INFO    wandb-upload_4:2640811 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/files/output.log
+2025-01-21 20:50:31,452 INFO    wandb-upload_3:2640811 [upload_job.py:push():130] Uploaded file /tmp/tmp82dkh83_wandb/0dnpg8u8-events.out.tfevents.1737510382.leonmkim-ROG-Strix-G15CS-G15CS.2640629.0
+2025-01-21 20:50:31,653 INFO    Thread-11 (_thread_body):2640811 [sender.py:transition_state():614] send defer: 11
+2025-01-21 20:50:31,653 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,653 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-21 20:50:31,653 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,653 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-21 20:50:31,653 INFO    SenderThread:2640811 [file_pusher.py:join():175] waiting for file pusher
+2025-01-21 20:50:31,654 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 12
+2025-01-21 20:50:31,654 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,655 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-21 20:50:31,655 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,655 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-21 20:50:31,655 INFO    SenderThread:2640811 [file_stream.py:finish():601] file stream finish called
+2025-01-21 20:50:31,867 INFO    SenderThread:2640811 [file_stream.py:finish():605] file stream finish is done
+2025-01-21 20:50:31,867 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 13
+2025-01-21 20:50:31,867 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,867 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-21 20:50:31,867 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,867 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-21 20:50:31,868 INFO    SenderThread:2640811 [sender.py:transition_state():614] send defer: 14
+2025-01-21 20:50:31,868 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: defer
+2025-01-21 20:50:31,868 INFO    HandlerThread:2640811 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-21 20:50:31,868 DEBUG   SenderThread:2640811 [sender.py:send():379] send: final
+2025-01-21 20:50:31,868 DEBUG   SenderThread:2640811 [sender.py:send():379] send: footer
+2025-01-21 20:50:31,868 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: defer
+2025-01-21 20:50:31,868 INFO    SenderThread:2640811 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-21 20:50:31,868 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:31,869 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,869 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: server_info
+2025-01-21 20:50:31,869 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-21 20:50:31,870 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: server_info
+2025-01-21 20:50:31,871 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-21 20:50:31,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-21 20:50:31,872 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-21 20:50:31,935 DEBUG   SenderThread:2640811 [sender.py:send_request():406] send_request: poll_exit
+2025-01-21 20:50:31,936 INFO    MainThread:2640811 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-21 20:50:31,936 INFO    MainThread:2640811 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-21 20:50:31,936 INFO    MainThread:2640811 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-21 20:50:31,937 DEBUG   HandlerThread:2640811 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-21 20:50:31,937 INFO    HandlerThread:2640811 [handler.py:finish():882] shutting down handler
+2025-01-21 20:50:32,870 INFO    WriterThread:2640811 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb
+2025-01-21 20:50:32,936 INFO    SenderThread:2640811 [sender.py:finish():1615] shutting down sender
+2025-01-21 20:50:32,936 INFO    SenderThread:2640811 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-21 20:50:32,936 INFO    SenderThread:2640811 [file_pusher.py:join():175] waiting for file pusher
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug.log b/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..ce02924333803090c8bcd5c18ae546a5f51016cf
--- /dev/null
+++ b/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Configure stats pid to 2640629
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/settings
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-21 20:46:21,763 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug.log
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/wandb/run-20250121_204621-rgws6h04/logs/debug-internal.log
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():569] calling init triggers
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04'}
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():619] starting backend
+2025-01-21 20:46:21,764 INFO    MainThread:2640629 [wandb_init.py:init():623] setting up manager
+2025-01-21 20:46:21,768 INFO    MainThread:2640629 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-21 20:46:21,769 INFO    MainThread:2640629 [wandb_init.py:init():631] backend started and connected
+2025-01-21 20:46:21,781 INFO    MainThread:2640629 [wandb_init.py:init():720] updated telemetry
+2025-01-21 20:46:21,787 INFO    MainThread:2640629 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-21 20:46:22,116 INFO    MainThread:2640629 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-21 20:46:22,155 INFO    MainThread:2640629 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-21 20:46:22,156 INFO    MainThread:2640629 [wandb_init.py:init():804] starting run threads in backend
+2025-01-21 20:46:22,528 INFO    MainThread:2640629 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-21 20:46:22,528 INFO    MainThread:2640629 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-21 20:46:22,528 INFO    MainThread:2640629 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-21 20:46:22,529 INFO    MainThread:2640629 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-21 20:46:22,530 INFO    MainThread:2640629 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-21 20:46:22,531 INFO    MainThread:2640629 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/rgws6h04/tb, True
+2025-01-21 20:46:26,108 INFO    MainThread:2640629 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-21 20:50:32,937 WARNING MsgRouterThr:2640629 [router.py:message_loop():77] message_loop has been closed
diff --git a/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb b/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..f795cc05abb27b9b059addb3b953d25cf5f03976
Binary files /dev/null and b/rgws6h04/wandb/run-20250121_204621-rgws6h04/run-rgws6h04.wandb differ